@@ -12,7 +12,7 @@ import { createLogger } from '@sim/logger'
1212import { sha256Hex } from '@sim/security/hash'
1313import { generateId , generateShortId } from '@sim/utils/id'
1414import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits'
15- import { and , asc , eq , gt , inArray , lt , lte , sql } from 'drizzle-orm'
15+ import { and , asc , eq , gt , inArray , lt , lte , notInArray , sql } from 'drizzle-orm'
1616import { OrchestrationError } from '@/lib/core/orchestration/types'
1717import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks'
1818import { deleteFiles , downloadFile , uploadFile } from '@/lib/uploads/core/storage-service'
@@ -247,31 +247,39 @@ export async function readStagedBlob(blob: StagedBlob): Promise<Buffer> {
247247/**
248248 * Deletes expired staged blobs, oldest first, up to `maxBatches` passes of {@link SWEEP_BATCH}:
249249 * each pass removes the objects, then the rows of the objects it removed. A row whose object could
250- * not be removed stays expired (moved to the back of the queue), so a later sweep retries it
251- * rather than orphaning the object. Returns how many rows it removed.
250+ * not be removed stays expired, moved to the back of the queue just before this sweep's start, so
251+ * any later sweep retries it (even one starting in the same millisecond) rather than orphaning the
252+ * object; this sweep skips the rows it already kept. Returns how many rows it removed.
252253 */
253254export async function sweepExpiredStagedBlobs ( maxBatches = 20 ) : Promise < number > {
254255 let removed = 0
255256 const sweepStart = new Date ( )
257+ const retryAt = new Date ( sweepStart . getTime ( ) - 1 )
258+ const kept : string [ ] = [ ]
256259 for ( let pass = 0 ; pass < maxBatches ; pass ++ ) {
257260 const expired = await db
258261 . select ( {
259262 id : workspaceSourceStagedBlob . id ,
260263 storageKey : workspaceSourceStagedBlob . storageKey ,
261264 } )
262265 . from ( workspaceSourceStagedBlob )
263- . where ( lt ( workspaceSourceStagedBlob . expiresAt , sweepStart ) )
266+ . where (
267+ and (
268+ lt ( workspaceSourceStagedBlob . expiresAt , sweepStart ) ,
269+ kept . length > 0 ? notInArray ( workspaceSourceStagedBlob . id , kept ) : undefined
270+ )
271+ )
264272 . orderBy ( asc ( workspaceSourceStagedBlob . expiresAt ) )
265273 . limit ( SWEEP_BATCH )
266274 if ( expired . length === 0 ) break
267275 const { failed } = await deleteFiles (
268276 expired . map ( ( row ) => row . storageKey ) ,
269277 STORAGE_CONTEXT
270278 )
271- const kept = new Set ( failed . map ( ( failure ) => failure . key ) )
272- if ( kept . size > 0 ) logger . warn ( 'Could not remove staged blob objects' , { failed } )
273- const gone = expired . filter ( ( row ) => ! kept . has ( row . storageKey ) ) . map ( ( row ) => row . id )
274- const stuck = expired . filter ( ( row ) => kept . has ( row . storageKey ) ) . map ( ( row ) => row . id )
279+ const failedKeys = new Set ( failed . map ( ( failure ) => failure . key ) )
280+ if ( failedKeys . size > 0 ) logger . warn ( 'Could not remove staged blob objects' , { failed } )
281+ const gone = expired . filter ( ( row ) => ! failedKeys . has ( row . storageKey ) ) . map ( ( row ) => row . id )
282+ const stuck = expired . filter ( ( row ) => failedKeys . has ( row . storageKey ) ) . map ( ( row ) => row . id )
275283 if ( gone . length > 0 ) {
276284 const rows = await db
277285 . delete ( workspaceSourceStagedBlob )
@@ -287,8 +295,14 @@ export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise<number>
287295 if ( stuck . length > 0 ) {
288296 await db
289297 . update ( workspaceSourceStagedBlob )
290- . set ( { expiresAt : sweepStart } )
291- . where ( inArray ( workspaceSourceStagedBlob . id , stuck ) )
298+ . set ( { expiresAt : retryAt } )
299+ . where (
300+ and (
301+ inArray ( workspaceSourceStagedBlob . id , stuck ) ,
302+ lt ( workspaceSourceStagedBlob . expiresAt , sweepStart )
303+ )
304+ )
305+ kept . push ( ...stuck )
292306 }
293307 if ( expired . length < SWEEP_BATCH ) break
294308 }
0 commit comments