Skip to content

Commit 126da70

Browse files
authored
fix(workspace-source): retry blobs a sweep kept even when the next sweep starts in the same millisecond (#10256)
1 parent 8d12171 commit 126da70

2 files changed

Lines changed: 57 additions & 10 deletions

File tree

‎apps/sim/lib/workspace-source/application/staging.integration.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,4 +302,37 @@ describe('workspace source staging', () => {
302302
expect(stored(row.storageKey)).toBe(false)
303303
}
304304
)
305+
306+
check(
307+
'a blob kept by one sweep is removed by the next even when both start in the same millisecond',
308+
async () => {
309+
const workspaceId = await newWorkspace()
310+
await stage(owner, workspaceId, [blob('kept within one millisecond')])
311+
await db
312+
.update(workspaceSourceStagedBlob)
313+
.set({ expiresAt: new Date(Date.now() - 1000) })
314+
.where(eq(workspaceSourceStagedBlob.workspaceId, workspaceId))
315+
const [row] = await rows(workspaceId)
316+
const folder = path.dirname(path.join(storageRoot, row.storageKey))
317+
// Only Date is faked: both sweeps read one start time, as back-to-back sweeps on a fast
318+
// database can.
319+
vi.useFakeTimers({ toFake: ['Date'], now: Date.now() })
320+
try {
321+
chmodSync(folder, 0o555)
322+
try {
323+
await sweepExpiredStagedBlobs()
324+
} finally {
325+
chmodSync(folder, 0o755)
326+
}
327+
expect((await rows(workspaceId)).map((left) => left.storageKey)).toEqual([row.storageKey])
328+
329+
await sweepExpiredStagedBlobs()
330+
} finally {
331+
vi.useRealTimers()
332+
}
333+
334+
expect(await rows(workspaceId)).toEqual([])
335+
expect(stored(row.storageKey)).toBe(false)
336+
}
337+
)
305338
})

‎apps/sim/lib/workspace-source/staging/store.ts‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import { createLogger } from '@sim/logger'
1212
import { sha256Hex } from '@sim/security/hash'
1313
import { generateId, generateShortId } from '@sim/utils/id'
1414
import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits'
15-
import { and, asc, eq, gt, inArray, lt, lte, sql } from 'drizzle-orm'
15+
import { and, asc, eq, gt, inArray, lt, lte, notInArray, sql } from 'drizzle-orm'
1616
import { OrchestrationError } from '@/lib/core/orchestration/types'
1717
import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks'
1818
import { deleteFiles, downloadFile, uploadFile } from '@/lib/uploads/core/storage-service'
@@ -247,31 +247,39 @@ export async function readStagedBlob(blob: StagedBlob): Promise<Buffer> {
247247
/**
248248
* Deletes expired staged blobs, oldest first, up to `maxBatches` passes of {@link SWEEP_BATCH}:
249249
* each pass removes the objects, then the rows of the objects it removed. A row whose object could
250-
* not be removed stays expired (moved to the back of the queue), so a later sweep retries it
251-
* rather than orphaning the object. Returns how many rows it removed.
250+
* not be removed stays expired, moved to the back of the queue just before this sweep's start, so
251+
* any later sweep retries it (even one starting in the same millisecond) rather than orphaning the
252+
* object; this sweep skips the rows it already kept. Returns how many rows it removed.
252253
*/
253254
export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise<number> {
254255
let removed = 0
255256
const sweepStart = new Date()
257+
const retryAt = new Date(sweepStart.getTime() - 1)
258+
const kept: string[] = []
256259
for (let pass = 0; pass < maxBatches; pass++) {
257260
const expired = await db
258261
.select({
259262
id: workspaceSourceStagedBlob.id,
260263
storageKey: workspaceSourceStagedBlob.storageKey,
261264
})
262265
.from(workspaceSourceStagedBlob)
263-
.where(lt(workspaceSourceStagedBlob.expiresAt, sweepStart))
266+
.where(
267+
and(
268+
lt(workspaceSourceStagedBlob.expiresAt, sweepStart),
269+
kept.length > 0 ? notInArray(workspaceSourceStagedBlob.id, kept) : undefined
270+
)
271+
)
264272
.orderBy(asc(workspaceSourceStagedBlob.expiresAt))
265273
.limit(SWEEP_BATCH)
266274
if (expired.length === 0) break
267275
const { failed } = await deleteFiles(
268276
expired.map((row) => row.storageKey),
269277
STORAGE_CONTEXT
270278
)
271-
const kept = new Set(failed.map((failure) => failure.key))
272-
if (kept.size > 0) logger.warn('Could not remove staged blob objects', { failed })
273-
const gone = expired.filter((row) => !kept.has(row.storageKey)).map((row) => row.id)
274-
const stuck = expired.filter((row) => kept.has(row.storageKey)).map((row) => row.id)
279+
const failedKeys = new Set(failed.map((failure) => failure.key))
280+
if (failedKeys.size > 0) logger.warn('Could not remove staged blob objects', { failed })
281+
const gone = expired.filter((row) => !failedKeys.has(row.storageKey)).map((row) => row.id)
282+
const stuck = expired.filter((row) => failedKeys.has(row.storageKey)).map((row) => row.id)
275283
if (gone.length > 0) {
276284
const rows = await db
277285
.delete(workspaceSourceStagedBlob)
@@ -287,8 +295,14 @@ export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise<number>
287295
if (stuck.length > 0) {
288296
await db
289297
.update(workspaceSourceStagedBlob)
290-
.set({ expiresAt: sweepStart })
291-
.where(inArray(workspaceSourceStagedBlob.id, stuck))
298+
.set({ expiresAt: retryAt })
299+
.where(
300+
and(
301+
inArray(workspaceSourceStagedBlob.id, stuck),
302+
lt(workspaceSourceStagedBlob.expiresAt, sweepStart)
303+
)
304+
)
305+
kept.push(...stuck)
292306
}
293307
if (expired.length < SWEEP_BATCH) break
294308
}

0 commit comments

Comments
 (0)