diff --git a/apps/sim/lib/billing/activity-rollup/pass.ts b/apps/sim/lib/billing/activity-rollup/pass.ts index 15c14cfc9b3..6d8a8908ef8 100644 --- a/apps/sim/lib/billing/activity-rollup/pass.ts +++ b/apps/sim/lib/billing/activity-rollup/pass.ts @@ -50,6 +50,7 @@ import { resetRollup, truncateRollupBatch, updateLeasedState, + WORKFLOW_ROLLUP_ID, } from '@/lib/billing/activity-rollup/state' import type { DbTransaction } from '@/lib/db/types' @@ -143,8 +144,9 @@ function startCoverage( } /** - * Once this many passes in a row have failed, a deferred hour counts as a retry, so a pass that - * keeps crashing while paging it backs it off instead of starting it again every tick. + * Once this many passes in a row have failed (thrown, or died holding the lease), a deferred hour + * counts as a retry, so a pass that keeps crashing while paging it backs it off instead of starting + * it again every tick. */ const CRASHING_PASSES = 3 @@ -663,14 +665,31 @@ export async function runActivityRollupPass( } } -/** Counts a pass that failed after taking the lease, and logs the run of failures loudly. */ +/** + * Counts a pass that failed after taking the lease and releases the lease in the same update, so a + * takeover never counts the failure again; logs the run of failures loudly. Matched on the owner + * alone, so a pass that overran its lease is still counted; one whose lease another pass took over + * was already counted by that takeover, and matches nothing. + */ async function recordFailedPass(owner: string, error: unknown): Promise { try { - const state = await updateLeasedState(owner, { - failedPasses: sql`${activityRollupState.failedPasses} + 1`, - }) + const [state] = await db + .update(activityRollupState) + .set({ + failedPasses: sql`${activityRollupState.failedPasses} + 1`, + leaseOwner: null, + leaseExpiresAt: null, + updatedAt: sql`now()`, + }) + .where( + and( + eq(activityRollupState.id, WORKFLOW_ROLLUP_ID), + eq(activityRollupState.leaseOwner, owner) + ) + ) + .returning({ failedPasses: activityRollupState.failedPasses }) logger.error('Activity rollup pass failed', { - consecutiveFailures: state.failedPasses, + consecutiveFailures: state?.failedPasses ?? null, error: getErrorMessage(error), }) } catch (countError) { diff --git a/apps/sim/lib/billing/activity-rollup/state.ts b/apps/sim/lib/billing/activity-rollup/state.ts index 068a5f2648f..ed546d001b2 100644 --- a/apps/sim/lib/billing/activity-rollup/state.ts +++ b/apps/sim/lib/billing/activity-rollup/state.ts @@ -52,6 +52,11 @@ export async function readRollupState( * Takes the pass lease, creating the state row on first use, so two passes never run at once: * Trigger.dev runs one at a time, but an inline pass and a Trigger.dev pass can meet while a * deployment changes how it runs them. Returns the state, or null when another pass holds it. + * + * Every pass releases the lease on its way out, so one taken over after it expired belonged to a + * pass that died holding it (killed, or out of memory) and is counted as a failed pass, like one + * that threw. A lease left behind for any other reason (its release failing) counts the same way, + * which at worst backs off a deferred hour one failure early. */ export async function acquireLease(owner: string): Promise { await db @@ -63,6 +68,7 @@ export async function acquireLease(owner: string): Promise !at || now.getTime() - at.getTime() >= MAX_FRESHNESS_LAG_MS if (lagging(freshThrough) || lagging(state.correctionsAt)) return null + const { lastEndScan } = state + if (!lastEndScan || now.getTime() - lastEndScan.getTime() >= MAX_LATE_END_SCAN_LAG_MS) { + return null + } if (window.from < coverageStart && !state.backfillCompletedAt) return null const from = new Date(Math.max(ceilHour(window.from).getTime(), coverageStart.getTime())) @@ -84,7 +103,7 @@ export function planRollupRead( /** * Reads the state and the rows it vouches for in one snapshot of the replica, so a pass that * commits mid-request can never pair new progress with old cells. Null when the rollup cannot - * serve the window. + * serve the window, or when more of its cells are marked than are worth reading live. */ function readInSnapshot( scope: ActivityScope, @@ -97,7 +116,15 @@ function readInSnapshot( { from: scope.start, to: scope.end }, new Date() ) - return plan ? read(tx, plan) : null + if (!plan) return null + if (plan.rollup && (await exceedsLiveMarkedCells(tx, scope, plan.rollup))) { + logger.info('Organization activity read too many marked cells; using the legacy read', { + organizationId: scope.organizationId, + maxLiveMarkedCells: MAX_LIVE_MARKED_CELLS, + }) + return null + } + return read(tx, plan) }, { isolationLevel: 'repeatable read', accessMode: 'read only' } ) @@ -124,16 +151,33 @@ function storedCells(scope: ActivityScope, span: InstantRange): SQL { * The rollup cells of the scope's workspaces that have a pending dirty mark, as * `(workspace_id, h)`: known to be stale until the mark is processed (or while it backs off after * failing), so reads take exactly those workspace-hours live and every other workspace's stored - * cell for the hour still serves. Read on the mark queue's key, in the read's snapshot. + * cell for the hour still serves. Read on the mark queue's key, in the read's snapshot. Each row is + * a distinct cell: the queue's key is the workspace-hour, and each mark joins one workspace. */ function markedCells(scope: ActivityScope, span: InstantRange): SQL { - return sql`SELECT DISTINCT d.workspace_id, d.bucket_start AS h FROM ${workspace} w + return sql`SELECT d.workspace_id, d.bucket_start AS h FROM ${workspace} w JOIN ${activityRollupDirty} d ON d.workspace_id = w.id AND d.bucket_start >= ${utcTimestamp(span.from)} AND d.bucket_start < ${utcTimestamp(span.to)} WHERE w.organization_id = ${scope.organizationId} ${scope.workspaceId ? sql`AND w.id = ${scope.workspaceId}` : sql``}` } +/** + * Whether the scope has more than {@link MAX_LIVE_MARKED_CELLS} marked cells in `span`, reading + * no further than one past the bound. + */ +async function exceedsLiveMarkedCells( + tx: DbTransaction, + scope: ActivityScope, + span: InstantRange +): Promise { + const [row] = await tx.execute<{ exceeds: boolean }>(sql` + SELECT count(*) > ${MAX_LIVE_MARKED_CELLS} AS exceeds + FROM (${markedCells(scope, span)} LIMIT ${MAX_LIVE_MARKED_CELLS + 1}) m + `) + return row?.exceeds === true +} + /** A stored cell `r` of workspace `w` whose workspace-hour is not among `marked` cells `m`. */ function notMarked(marked: SQL): SQL { return sql`NOT EXISTS (SELECT 1 FROM (${marked}) m WHERE m.workspace_id = w.id AND m.h = r.bucket_start)` diff --git a/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts b/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts index 4bad9895bed..62a9a11340d 100644 --- a/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts +++ b/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts @@ -923,6 +923,29 @@ describe('organization activity rollup', () => { expect((await readRollupState())?.failedPasses).toBe(0) }, 300_000) + it('counts a pass that died holding the lease as failed, so a deferred hour backs off', async () => { + const hour = await reopenLastSettledHour() + await connection.unsafe( + `INSERT INTO activity_rollup_blocked VALUES ($1::timestamp, 0, now())`, + [hour] + ) + // Two passes threw; a third was killed paging the hour, so its lease was never released. + await connection.unsafe( + `UPDATE activity_rollup_state SET failed_passes = 2, lease_owner = 'killed-pass', + lease_expires_at = now() - interval '1 second'` + ) + await gateHour(hour, `RAISE EXCEPTION 'Synthetic crash';`) + try { + await expect(runPass()).rejects.toThrow() + expect(await blockedAttempts(hour)).toBe(1) + } finally { + await ungateHour() + } + await connection.unsafe(`UPDATE activity_rollup_blocked SET retry_at = now()`) + expect(await runPass()).toMatchObject({ finalizedHours: 1, blockedHours: 0 }) + expect((await readRollupState())?.failedPasses).toBe(0) + }, 300_000) + it('commits a retry backoff before the retry, so a retry that dies still backs off', async () => { const hour = await reopenLastSettledHour() await connection.unsafe( @@ -1119,6 +1142,41 @@ describe('organization activity rollup', () => { await expectEquivalent() }, 300_000) + it('stops serving the rollup while the late-end scan has fallen behind', async () => { + await connection.unsafe( + `UPDATE activity_rollup_state SET last_end_scan = (now() AT TIME ZONE 'UTC') - interval '1 hour'` + ) + expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).toBeNull() + expect( + await readRollupActivityBreakdown(scopeOf(longWindow()), 'workflow', 'runs', 0) + ).toBeNull() + await runPass() + expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).not.toBeNull() + await expectEquivalent() + }, 300_000) + + it('uses the legacy read when more cells are marked than are worth reading live', async () => { + const state = await readRollupState() + if (!state?.freshThrough) throw new Error('No coverage') + // Past the read's bound of 5,000 marked cells: organization A's workspaces, inside the long + // window, due only after the test so no pass takes them. + await connection`INSERT INTO activity_rollup_dirty (workspace_id, bucket_start, due_at) + SELECT w, ${state.freshThrough.toISOString()}::timestamptz AT TIME ZONE 'UTC' - h * interval '1 hour', + now() + interval '1 day' + FROM unnest(ARRAY['ws1', 'ws2', 'ws4']) AS w, generate_series(1, 1700) AS h + ON CONFLICT (workspace_id, bucket_start) DO NOTHING` + try { + expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).toBeNull() + expect( + await readRollupActivityBreakdown(scopeOf(longWindow()), 'workflow', 'runs', 0) + ).toBeNull() + } finally { + await connection`DELETE FROM activity_rollup_dirty WHERE due_at > now() + interval '1 hour'` + } + expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).not.toBeNull() + await expectEquivalent() + }, 300_000) + it('backs off an hour too heavy to read and keeps the rest of the pass running', async () => { const [{ hour }] = await connection.unsafe<{ hour: string }[]>( `SELECT to_char(h, 'YYYY-MM-DD HH24:MI:SS') AS hour FROM ( diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 2585f6a232e..33d4ba270a5 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -807,7 +807,10 @@ export const activityRollupState = pgTable('activity_rollup_state', { * rebuilds the rollup. */ maintainedAt: timestamp('maintained_at', { withTimezone: true }), - /** Passes in a row that failed after taking the lease; reset by the next pass that completes. */ + /** + * Passes in a row that failed after taking the lease, by throwing or by dying with it held; + * reset by the next pass that completes. + */ failedPasses: integer('failed_passes').notNull().default(0), /** * When a pass last ran both correction queues (dirty marks and in-flight rechecks) to the end of