Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions apps/sim/lib/workspace-source/application/staging.integration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -302,4 +302,37 @@ describe('workspace source staging', () => {
expect(stored(row.storageKey)).toBe(false)
}
)

check(
'a blob kept by one sweep is removed by the next even when both start in the same millisecond',
async () => {
const workspaceId = await newWorkspace()
await stage(owner, workspaceId, [blob('kept within one millisecond')])
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
await db
.update(workspaceSourceStagedBlob)
.set({ expiresAt: new Date(Date.now() - 1000) })
.where(eq(workspaceSourceStagedBlob.workspaceId, workspaceId))
const [row] = await rows(workspaceId)
const folder = path.dirname(path.join(storageRoot, row.storageKey))
// Only Date is faked: both sweeps read one start time, as back-to-back sweeps on a fast
// database can.
vi.useFakeTimers({ toFake: ['Date'], now: Date.now() })
try {
chmodSync(folder, 0o555)
try {
await sweepExpiredStagedBlobs()
} finally {
chmodSync(folder, 0o755)
}
expect((await rows(workspaceId)).map((left) => left.storageKey)).toEqual([row.storageKey])

await sweepExpiredStagedBlobs()
} finally {
vi.useRealTimers()
}

expect(await rows(workspaceId)).toEqual([])
expect(stored(row.storageKey)).toBe(false)
}
)
})
34 changes: 24 additions & 10 deletions apps/sim/lib/workspace-source/staging/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { createLogger } from '@sim/logger'
import { sha256Hex } from '@sim/security/hash'
import { generateId, generateShortId } from '@sim/utils/id'
import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits'
import { and, asc, eq, gt, inArray, lt, lte, sql } from 'drizzle-orm'
import { and, asc, eq, gt, inArray, lt, lte, notInArray, sql } from 'drizzle-orm'
import { OrchestrationError } from '@/lib/core/orchestration/types'
import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks'
import { deleteFiles, downloadFile, uploadFile } from '@/lib/uploads/core/storage-service'
Expand Down Expand Up @@ -247,31 +247,39 @@ export async function readStagedBlob(blob: StagedBlob): Promise<Buffer> {
/**
* Deletes expired staged blobs, oldest first, up to `maxBatches` passes of {@link SWEEP_BATCH}:
* each pass removes the objects, then the rows of the objects it removed. A row whose object could
* not be removed stays expired (moved to the back of the queue), so a later sweep retries it
* rather than orphaning the object. Returns how many rows it removed.
* not be removed stays expired, moved to the back of the queue just before this sweep's start, so
* any later sweep retries it (even one starting in the same millisecond) rather than orphaning the
* object; this sweep skips the rows it already kept. Returns how many rows it removed.
*/
export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise<number> {
let removed = 0
const sweepStart = new Date()
const retryAt = new Date(sweepStart.getTime() - 1)
const kept: string[] = []
for (let pass = 0; pass < maxBatches; pass++) {
const expired = await db
.select({
id: workspaceSourceStagedBlob.id,
storageKey: workspaceSourceStagedBlob.storageKey,
})
.from(workspaceSourceStagedBlob)
.where(lt(workspaceSourceStagedBlob.expiresAt, sweepStart))
.where(
and(
lt(workspaceSourceStagedBlob.expiresAt, sweepStart),
kept.length > 0 ? notInArray(workspaceSourceStagedBlob.id, kept) : undefined
)
)
.orderBy(asc(workspaceSourceStagedBlob.expiresAt))
.limit(SWEEP_BATCH)
if (expired.length === 0) break
const { failed } = await deleteFiles(
expired.map((row) => row.storageKey),
STORAGE_CONTEXT
)
const kept = new Set(failed.map((failure) => failure.key))
if (kept.size > 0) logger.warn('Could not remove staged blob objects', { failed })
const gone = expired.filter((row) => !kept.has(row.storageKey)).map((row) => row.id)
const stuck = expired.filter((row) => kept.has(row.storageKey)).map((row) => row.id)
const failedKeys = new Set(failed.map((failure) => failure.key))
if (failedKeys.size > 0) logger.warn('Could not remove staged blob objects', { failed })
const gone = expired.filter((row) => !failedKeys.has(row.storageKey)).map((row) => row.id)
const stuck = expired.filter((row) => failedKeys.has(row.storageKey)).map((row) => row.id)
if (gone.length > 0) {
const rows = await db
.delete(workspaceSourceStagedBlob)
Expand All @@ -287,8 +295,14 @@ export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise<number>
if (stuck.length > 0) {
await db
.update(workspaceSourceStagedBlob)
.set({ expiresAt: sweepStart })
.where(inArray(workspaceSourceStagedBlob.id, stuck))
.set({ expiresAt: retryAt })
.where(
and(
inArray(workspaceSourceStagedBlob.id, stuck),
lt(workspaceSourceStagedBlob.expiresAt, sweepStart)
)
)
kept.push(...stuck)
}
if (expired.length < SWEEP_BATCH) break
}
Expand Down
Loading