diff --git a/apps/sim/lib/workspace-source/application/staging.integration.ts b/apps/sim/lib/workspace-source/application/staging.integration.ts index c580f55e9d6..1fd331b9802 100644 --- a/apps/sim/lib/workspace-source/application/staging.integration.ts +++ b/apps/sim/lib/workspace-source/application/staging.integration.ts @@ -302,4 +302,37 @@ describe('workspace source staging', () => { expect(stored(row.storageKey)).toBe(false) } ) + + check( + 'a blob kept by one sweep is removed by the next even when both start in the same millisecond', + async () => { + const workspaceId = await newWorkspace() + await stage(owner, workspaceId, [blob('kept within one millisecond')]) + await db + .update(workspaceSourceStagedBlob) + .set({ expiresAt: new Date(Date.now() - 1000) }) + .where(eq(workspaceSourceStagedBlob.workspaceId, workspaceId)) + const [row] = await rows(workspaceId) + const folder = path.dirname(path.join(storageRoot, row.storageKey)) + // Only Date is faked: both sweeps read one start time, as back-to-back sweeps on a fast + // database can. + vi.useFakeTimers({ toFake: ['Date'], now: Date.now() }) + try { + chmodSync(folder, 0o555) + try { + await sweepExpiredStagedBlobs() + } finally { + chmodSync(folder, 0o755) + } + expect((await rows(workspaceId)).map((left) => left.storageKey)).toEqual([row.storageKey]) + + await sweepExpiredStagedBlobs() + } finally { + vi.useRealTimers() + } + + expect(await rows(workspaceId)).toEqual([]) + expect(stored(row.storageKey)).toBe(false) + } + ) }) diff --git a/apps/sim/lib/workspace-source/staging/store.ts b/apps/sim/lib/workspace-source/staging/store.ts index d2bd7a7548d..9e86bc55a30 100644 --- a/apps/sim/lib/workspace-source/staging/store.ts +++ b/apps/sim/lib/workspace-source/staging/store.ts @@ -12,7 +12,7 @@ import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { generateId, generateShortId } from '@sim/utils/id' import { WORKSPACE_SOURCE_LIMITS } from '@sim/workspace-code/limits' -import { and, asc, eq, gt, inArray, lt, lte, sql } from 'drizzle-orm' +import { and, asc, eq, gt, inArray, lt, lte, notInArray, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { acquireAdvisoryXactLock } from '@/lib/db/advisory-locks' import { deleteFiles, downloadFile, uploadFile } from '@/lib/uploads/core/storage-service' @@ -247,12 +247,15 @@ export async function readStagedBlob(blob: StagedBlob): Promise { /** * Deletes expired staged blobs, oldest first, up to `maxBatches` passes of {@link SWEEP_BATCH}: * each pass removes the objects, then the rows of the objects it removed. A row whose object could - * not be removed stays expired (moved to the back of the queue), so a later sweep retries it - * rather than orphaning the object. Returns how many rows it removed. + * not be removed stays expired, moved to the back of the queue just before this sweep's start, so + * any later sweep retries it (even one starting in the same millisecond) rather than orphaning the + * object; this sweep skips the rows it already kept. Returns how many rows it removed. */ export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise { let removed = 0 const sweepStart = new Date() + const retryAt = new Date(sweepStart.getTime() - 1) + const kept: string[] = [] for (let pass = 0; pass < maxBatches; pass++) { const expired = await db .select({ @@ -260,7 +263,12 @@ export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise storageKey: workspaceSourceStagedBlob.storageKey, }) .from(workspaceSourceStagedBlob) - .where(lt(workspaceSourceStagedBlob.expiresAt, sweepStart)) + .where( + and( + lt(workspaceSourceStagedBlob.expiresAt, sweepStart), + kept.length > 0 ? notInArray(workspaceSourceStagedBlob.id, kept) : undefined + ) + ) .orderBy(asc(workspaceSourceStagedBlob.expiresAt)) .limit(SWEEP_BATCH) if (expired.length === 0) break @@ -268,10 +276,10 @@ export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise expired.map((row) => row.storageKey), STORAGE_CONTEXT ) - const kept = new Set(failed.map((failure) => failure.key)) - if (kept.size > 0) logger.warn('Could not remove staged blob objects', { failed }) - const gone = expired.filter((row) => !kept.has(row.storageKey)).map((row) => row.id) - const stuck = expired.filter((row) => kept.has(row.storageKey)).map((row) => row.id) + const failedKeys = new Set(failed.map((failure) => failure.key)) + if (failedKeys.size > 0) logger.warn('Could not remove staged blob objects', { failed }) + const gone = expired.filter((row) => !failedKeys.has(row.storageKey)).map((row) => row.id) + const stuck = expired.filter((row) => failedKeys.has(row.storageKey)).map((row) => row.id) if (gone.length > 0) { const rows = await db .delete(workspaceSourceStagedBlob) @@ -287,8 +295,14 @@ export async function sweepExpiredStagedBlobs(maxBatches = 20): Promise if (stuck.length > 0) { await db .update(workspaceSourceStagedBlob) - .set({ expiresAt: sweepStart }) - .where(inArray(workspaceSourceStagedBlob.id, stuck)) + .set({ expiresAt: retryAt }) + .where( + and( + inArray(workspaceSourceStagedBlob.id, stuck), + lt(workspaceSourceStagedBlob.expiresAt, sweepStart) + ) + ) + kept.push(...stuck) } if (expired.length < SWEEP_BATCH) break }