From 76768f2539722940010879ce632afb1bfcb54701 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 08:57:31 -0800 Subject: [PATCH 1/3] fix(usage): gate rollup reads on the late-end scan and marked-cell count, count dead passes Co-Authored-By: Claude Opus 5.5 --- apps/sim/lib/billing/activity-rollup/pass.ts | 5 +- apps/sim/lib/billing/activity-rollup/state.ts | 5 ++ .../organization-activity-rollup-queries.ts | 49 +++++++++++++++- ...rganization-activity-rollup.integration.ts | 58 +++++++++++++++++++ packages/db/schema.ts | 5 +- 5 files changed, 116 insertions(+), 6 deletions(-) diff --git a/apps/sim/lib/billing/activity-rollup/pass.ts b/apps/sim/lib/billing/activity-rollup/pass.ts index 15c14cfc9b3..d8e8508ea00 100644 --- a/apps/sim/lib/billing/activity-rollup/pass.ts +++ b/apps/sim/lib/billing/activity-rollup/pass.ts @@ -143,8 +143,9 @@ function startCoverage( } /** - * Once this many passes in a row have failed, a deferred hour counts as a retry, so a pass that - * keeps crashing while paging it backs it off instead of starting it again every tick. + * Once this many passes in a row have failed (thrown, or died holding the lease), a deferred hour + * counts as a retry, so a pass that keeps crashing while paging it backs it off instead of starting + * it again every tick. */ const CRASHING_PASSES = 3 diff --git a/apps/sim/lib/billing/activity-rollup/state.ts b/apps/sim/lib/billing/activity-rollup/state.ts index 068a5f2648f..5797ff7890f 100644 --- a/apps/sim/lib/billing/activity-rollup/state.ts +++ b/apps/sim/lib/billing/activity-rollup/state.ts @@ -52,6 +52,10 @@ export async function readRollupState( * Takes the pass lease, creating the state row on first use, so two passes never run at once: * Trigger.dev runs one at a time, but an inline pass and a Trigger.dev pass can meet while a * deployment changes how it runs them. Returns the state, or null when another pass holds it. + * + * Every pass releases the lease on its way out, so one taken over after it expired belonged to a + * pass that died holding it (killed, or out of memory) and is counted as a failed pass, like one + * that threw. */ export async function acquireLease(owner: string): Promise { await db @@ -63,6 +67,7 @@ export async function acquireLease(owner: string): Promise !at || now.getTime() - at.getTime() >= MAX_FRESHNESS_LAG_MS if (lagging(freshThrough) || lagging(state.correctionsAt)) return null + const { lastEndScan } = state + if (!lastEndScan || now.getTime() - lastEndScan.getTime() >= MAX_LATE_END_SCAN_LAG_MS) { + return null + } if (window.from < coverageStart && !state.backfillCompletedAt) return null const from = new Date(Math.max(ceilHour(window.from).getTime(), coverageStart.getTime())) @@ -84,7 +103,7 @@ export function planRollupRead( /** * Reads the state and the rows it vouches for in one snapshot of the replica, so a pass that * commits mid-request can never pair new progress with old cells. Null when the rollup cannot - * serve the window. + * serve the window, or when more of its cells are marked than are worth reading live. */ function readInSnapshot( scope: ActivityScope, @@ -97,7 +116,15 @@ function readInSnapshot( { from: scope.start, to: scope.end }, new Date() ) - return plan ? read(tx, plan) : null + if (!plan) return null + if (plan.rollup && (await exceedsLiveMarkedCells(tx, scope, plan.rollup))) { + logger.warn('Organization activity read too many marked cells; using the legacy read', { + organizationId: scope.organizationId, + maxLiveMarkedCells: MAX_LIVE_MARKED_CELLS, + }) + return null + } + return read(tx, plan) }, { isolationLevel: 'repeatable read', accessMode: 'read only' } ) @@ -134,6 +161,22 @@ function markedCells(scope: ActivityScope, span: InstantRange): SQL { ${scope.workspaceId ? sql`AND w.id = ${scope.workspaceId}` : sql``}` } +/** + * Whether the scope has more than {@link MAX_LIVE_MARKED_CELLS} marked cells in `span`, counted + * no further than one past the bound. + */ +async function exceedsLiveMarkedCells( + tx: DbTransaction, + scope: ActivityScope, + span: InstantRange +): Promise { + const [row] = await tx.execute<{ exceeds: boolean }>(sql` + SELECT count(*) > ${MAX_LIVE_MARKED_CELLS} AS exceeds + FROM (${markedCells(scope, span)} LIMIT ${MAX_LIVE_MARKED_CELLS + 1}) m + `) + return row?.exceeds === true +} + /** A stored cell `r` of workspace `w` whose workspace-hour is not among `marked` cells `m`. */ function notMarked(marked: SQL): SQL { return sql`NOT EXISTS (SELECT 1 FROM (${marked}) m WHERE m.workspace_id = w.id AND m.h = r.bucket_start)` diff --git a/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts b/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts index 4bad9895bed..62a9a11340d 100644 --- a/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts +++ b/apps/sim/lib/billing/core/organization-activity-rollup.integration.ts @@ -923,6 +923,29 @@ describe('organization activity rollup', () => { expect((await readRollupState())?.failedPasses).toBe(0) }, 300_000) + it('counts a pass that died holding the lease as failed, so a deferred hour backs off', async () => { + const hour = await reopenLastSettledHour() + await connection.unsafe( + `INSERT INTO activity_rollup_blocked VALUES ($1::timestamp, 0, now())`, + [hour] + ) + // Two passes threw; a third was killed paging the hour, so its lease was never released. + await connection.unsafe( + `UPDATE activity_rollup_state SET failed_passes = 2, lease_owner = 'killed-pass', + lease_expires_at = now() - interval '1 second'` + ) + await gateHour(hour, `RAISE EXCEPTION 'Synthetic crash';`) + try { + await expect(runPass()).rejects.toThrow() + expect(await blockedAttempts(hour)).toBe(1) + } finally { + await ungateHour() + } + await connection.unsafe(`UPDATE activity_rollup_blocked SET retry_at = now()`) + expect(await runPass()).toMatchObject({ finalizedHours: 1, blockedHours: 0 }) + expect((await readRollupState())?.failedPasses).toBe(0) + }, 300_000) + it('commits a retry backoff before the retry, so a retry that dies still backs off', async () => { const hour = await reopenLastSettledHour() await connection.unsafe( @@ -1119,6 +1142,41 @@ describe('organization activity rollup', () => { await expectEquivalent() }, 300_000) + it('stops serving the rollup while the late-end scan has fallen behind', async () => { + await connection.unsafe( + `UPDATE activity_rollup_state SET last_end_scan = (now() AT TIME ZONE 'UTC') - interval '1 hour'` + ) + expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).toBeNull() + expect( + await readRollupActivityBreakdown(scopeOf(longWindow()), 'workflow', 'runs', 0) + ).toBeNull() + await runPass() + expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).not.toBeNull() + await expectEquivalent() + }, 300_000) + + it('uses the legacy read when more cells are marked than are worth reading live', async () => { + const state = await readRollupState() + if (!state?.freshThrough) throw new Error('No coverage') + // Past the read's bound of 5,000 marked cells: organization A's workspaces, inside the long + // window, due only after the test so no pass takes them. + await connection`INSERT INTO activity_rollup_dirty (workspace_id, bucket_start, due_at) + SELECT w, ${state.freshThrough.toISOString()}::timestamptz AT TIME ZONE 'UTC' - h * interval '1 hour', + now() + interval '1 day' + FROM unnest(ARRAY['ws1', 'ws2', 'ws4']) AS w, generate_series(1, 1700) AS h + ON CONFLICT (workspace_id, bucket_start) DO NOTHING` + try { + expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).toBeNull() + expect( + await readRollupActivityBreakdown(scopeOf(longWindow()), 'workflow', 'runs', 0) + ).toBeNull() + } finally { + await connection`DELETE FROM activity_rollup_dirty WHERE due_at > now() + interval '1 hour'` + } + expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).not.toBeNull() + await expectEquivalent() + }, 300_000) + it('backs off an hour too heavy to read and keeps the rest of the pass running', async () => { const [{ hour }] = await connection.unsafe<{ hour: string }[]>( `SELECT to_char(h, 'YYYY-MM-DD HH24:MI:SS') AS hour FROM ( diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 2585f6a232e..33d4ba270a5 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -807,7 +807,10 @@ export const activityRollupState = pgTable('activity_rollup_state', { * rebuilds the rollup. */ maintainedAt: timestamp('maintained_at', { withTimezone: true }), - /** Passes in a row that failed after taking the lease; reset by the next pass that completes. */ + /** + * Passes in a row that failed after taking the lease, by throwing or by dying with it held; + * reset by the next pass that completes. + */ failedPasses: integer('failed_passes').notNull().default(0), /** * When a pass last ran both correction queues (dirty marks and in-flight rechecks) to the end of From fe10138221edec5a1c9c4082c1b29d7517e27e19 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 09:05:26 -0800 Subject: [PATCH 2/3] fix(usage): count a failed pass whose lease overran, log the marked-cell fallback at info Co-Authored-By: Claude Opus 5.5 --- apps/sim/lib/billing/activity-rollup/pass.ts | 22 ++++++++++++++----- apps/sim/lib/billing/activity-rollup/state.ts | 3 ++- .../organization-activity-rollup-queries.ts | 6 ++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/apps/sim/lib/billing/activity-rollup/pass.ts b/apps/sim/lib/billing/activity-rollup/pass.ts index d8e8508ea00..14aae768ed7 100644 --- a/apps/sim/lib/billing/activity-rollup/pass.ts +++ b/apps/sim/lib/billing/activity-rollup/pass.ts @@ -50,6 +50,7 @@ import { resetRollup, truncateRollupBatch, updateLeasedState, + WORKFLOW_ROLLUP_ID, } from '@/lib/billing/activity-rollup/state' import type { DbTransaction } from '@/lib/db/types' @@ -664,14 +665,25 @@ export async function runActivityRollupPass( } } -/** Counts a pass that failed after taking the lease, and logs the run of failures loudly. */ +/** + * Counts a pass that failed after taking the lease, and logs the run of failures loudly. Matched on + * the owner alone, so a pass that overran its lease is still counted; one whose lease another pass + * took over was already counted by that takeover, and matches nothing. + */ async function recordFailedPass(owner: string, error: unknown): Promise { try { - const state = await updateLeasedState(owner, { - failedPasses: sql`${activityRollupState.failedPasses} + 1`, - }) + const [state] = await db + .update(activityRollupState) + .set({ failedPasses: sql`${activityRollupState.failedPasses} + 1`, updatedAt: sql`now()` }) + .where( + and( + eq(activityRollupState.id, WORKFLOW_ROLLUP_ID), + eq(activityRollupState.leaseOwner, owner) + ) + ) + .returning({ failedPasses: activityRollupState.failedPasses }) logger.error('Activity rollup pass failed', { - consecutiveFailures: state.failedPasses, + consecutiveFailures: state?.failedPasses ?? null, error: getErrorMessage(error), }) } catch (countError) { diff --git a/apps/sim/lib/billing/activity-rollup/state.ts b/apps/sim/lib/billing/activity-rollup/state.ts index 5797ff7890f..ed546d001b2 100644 --- a/apps/sim/lib/billing/activity-rollup/state.ts +++ b/apps/sim/lib/billing/activity-rollup/state.ts @@ -55,7 +55,8 @@ export async function readRollupState( * * Every pass releases the lease on its way out, so one taken over after it expired belonged to a * pass that died holding it (killed, or out of memory) and is counted as a failed pass, like one - * that threw. + * that threw. A lease left behind for any other reason (its release failing) counts the same way, + * which at worst backs off a deferred hour one failure early. */ export async function acquireLease(owner: string): Promise { await db diff --git a/apps/sim/lib/billing/core/organization-activity-rollup-queries.ts b/apps/sim/lib/billing/core/organization-activity-rollup-queries.ts index 3a3d5a8023f..c85bbf1f731 100644 --- a/apps/sim/lib/billing/core/organization-activity-rollup-queries.ts +++ b/apps/sim/lib/billing/core/organization-activity-rollup-queries.ts @@ -48,8 +48,8 @@ const MAX_FRESHNESS_LAG_MS = 6 * HOUR_MS /** * Reads use the rollup only while the late-end scan is this recent: a run that ends late leaves * its stored cell stale until the scan marks it. The scan trails now by its margin plus the time - * between passes, minutes in all, so this bound is only crossed when the scan is failing or - * catching up a backlog. + * between passes (longer while a heavy-hour retry holds the lease), a quarter hour at most, so + * this bound is only crossed when the scan is failing or catching up a backlog. */ const MAX_LATE_END_SCAN_LAG_MS = HOUR_MS / 2 @@ -118,7 +118,7 @@ function readInSnapshot( ) if (!plan) return null if (plan.rollup && (await exceedsLiveMarkedCells(tx, scope, plan.rollup))) { - logger.warn('Organization activity read too many marked cells; using the legacy read', { + logger.info('Organization activity read too many marked cells; using the legacy read', { organizationId: scope.organizationId, maxLiveMarkedCells: MAX_LIVE_MARKED_CELLS, }) From 2791a501ee326b55f3dd8a3bd552bd44575a2fa5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 19 Nov 2026 09:45:05 -0800 Subject: [PATCH 3/3] fix(usage): bound the marked-cell count's scan, release the lease when counting a failed pass Co-Authored-By: Claude Opus 5.5 --- apps/sim/lib/billing/activity-rollup/pass.ts | 14 ++++++++++---- .../core/organization-activity-rollup-queries.ts | 7 ++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/apps/sim/lib/billing/activity-rollup/pass.ts b/apps/sim/lib/billing/activity-rollup/pass.ts index 14aae768ed7..6d8a8908ef8 100644 --- a/apps/sim/lib/billing/activity-rollup/pass.ts +++ b/apps/sim/lib/billing/activity-rollup/pass.ts @@ -666,15 +666,21 @@ export async function runActivityRollupPass( } /** - * Counts a pass that failed after taking the lease, and logs the run of failures loudly. Matched on - * the owner alone, so a pass that overran its lease is still counted; one whose lease another pass - * took over was already counted by that takeover, and matches nothing. + * Counts a pass that failed after taking the lease and releases the lease in the same update, so a + * takeover never counts the failure again; logs the run of failures loudly. Matched on the owner + * alone, so a pass that overran its lease is still counted; one whose lease another pass took over + * was already counted by that takeover, and matches nothing. */ async function recordFailedPass(owner: string, error: unknown): Promise { try { const [state] = await db .update(activityRollupState) - .set({ failedPasses: sql`${activityRollupState.failedPasses} + 1`, updatedAt: sql`now()` }) + .set({ + failedPasses: sql`${activityRollupState.failedPasses} + 1`, + leaseOwner: null, + leaseExpiresAt: null, + updatedAt: sql`now()`, + }) .where( and( eq(activityRollupState.id, WORKFLOW_ROLLUP_ID), diff --git a/apps/sim/lib/billing/core/organization-activity-rollup-queries.ts b/apps/sim/lib/billing/core/organization-activity-rollup-queries.ts index c85bbf1f731..459940c5773 100644 --- a/apps/sim/lib/billing/core/organization-activity-rollup-queries.ts +++ b/apps/sim/lib/billing/core/organization-activity-rollup-queries.ts @@ -151,10 +151,11 @@ function storedCells(scope: ActivityScope, span: InstantRange): SQL { * The rollup cells of the scope's workspaces that have a pending dirty mark, as * `(workspace_id, h)`: known to be stale until the mark is processed (or while it backs off after * failing), so reads take exactly those workspace-hours live and every other workspace's stored - * cell for the hour still serves. Read on the mark queue's key, in the read's snapshot. + * cell for the hour still serves. Read on the mark queue's key, in the read's snapshot. Each row is + * a distinct cell: the queue's key is the workspace-hour, and each mark joins one workspace. */ function markedCells(scope: ActivityScope, span: InstantRange): SQL { - return sql`SELECT DISTINCT d.workspace_id, d.bucket_start AS h FROM ${workspace} w + return sql`SELECT d.workspace_id, d.bucket_start AS h FROM ${workspace} w JOIN ${activityRollupDirty} d ON d.workspace_id = w.id AND d.bucket_start >= ${utcTimestamp(span.from)} AND d.bucket_start < ${utcTimestamp(span.to)} WHERE w.organization_id = ${scope.organizationId} @@ -162,7 +163,7 @@ function markedCells(scope: ActivityScope, span: InstantRange): SQL { } /** - * Whether the scope has more than {@link MAX_LIVE_MARKED_CELLS} marked cells in `span`, counted + * Whether the scope has more than {@link MAX_LIVE_MARKED_CELLS} marked cells in `span`, reading * no further than one past the bound. */ async function exceedsLiveMarkedCells(