Skip to content

Commit 3c74e17

Browse files
waleedlatif1claude
andcommitted
fix(usage): gate rollup reads on the late-end scan and marked-cell count, count dead passes
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 27a73ac commit 3c74e17

5 files changed

Lines changed: 116 additions & 6 deletions

File tree

‎apps/sim/lib/billing/activity-rollup/pass.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,9 @@ function startCoverage(
143143
}
144144

145145
/**
146-
* Once this many passes in a row have failed, a deferred hour counts as a retry, so a pass that
147-
* keeps crashing while paging it backs it off instead of starting it again every tick.
146+
* Once this many passes in a row have failed (thrown, or died holding the lease), a deferred hour
147+
* counts as a retry, so a pass that keeps crashing while paging it backs it off instead of starting
148+
* it again every tick.
148149
*/
149150
const CRASHING_PASSES = 3
150151

‎apps/sim/lib/billing/activity-rollup/state.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ export async function readRollupState(
5252
* Takes the pass lease, creating the state row on first use, so two passes never run at once:
5353
* Trigger.dev runs one at a time, but an inline pass and a Trigger.dev pass can meet while a
5454
* deployment changes how it runs them. Returns the state, or null when another pass holds it.
55+
*
56+
* Every pass releases the lease on its way out, so one taken over after it expired belonged to a
57+
* pass that died holding it (killed, or out of memory) and is counted as a failed pass, like one
58+
* that threw.
5559
*/
5660
export async function acquireLease(owner: string): Promise<ActivityRollupState | null> {
5761
await db
@@ -63,6 +67,7 @@ export async function acquireLease(owner: string): Promise<ActivityRollupState |
6367
.set({
6468
leaseOwner: owner,
6569
leaseExpiresAt: sql`now() + ${msInterval(LEASE_MS)}`,
70+
failedPasses: sql`${activityRollupState.failedPasses} + CASE WHEN ${activityRollupState.leaseOwner} IS NULL THEN 0 ELSE 1 END`,
6671
updatedAt: sql`now()`,
6772
})
6873
.where(

‎apps/sim/lib/billing/core/organization-activity-rollup-queries.ts‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,20 @@ const logger = createLogger('OrganizationActivityRollupQueries')
4545
*/
4646
const MAX_FRESHNESS_LAG_MS = 6 * HOUR_MS
4747

48+
/**
49+
* Reads use the rollup only while the late-end scan is this recent: a run that ends late leaves
50+
* its stored cell stale until the scan marks it. The scan trails now by its margin plus the time
51+
* between passes, minutes in all, so this bound is only crossed when the scan is failing or
52+
* catching up a backlog.
53+
*/
54+
const MAX_LATE_END_SCAN_LAG_MS = HOUR_MS / 2
55+
56+
/**
57+
* Each marked cell is read live as its own workspace-hour; past this many in a window, those
58+
* reads approach the cost of the legacy read itself, so the legacy read serves instead.
59+
*/
60+
const MAX_LIVE_MARKED_CELLS = 5_000
61+
4862
/** Which part of a window the rollup serves, and the instants read live from the source. */
4963
export interface RollupReadPlan {
5064
/** Whole UTC hours, all computed. */
@@ -56,7 +70,8 @@ export interface RollupReadPlan {
5670
/**
5771
* Splits `window` between the rollup and the source, or returns null when the rollup must not
5872
* serve it at all: another version's cells, no coverage yet, a stalled pass, correction queues
59-
* (dirty marks and in-flight rechecks) that no pass has run lately, or a window reaching before a
73+
* (dirty marks and in-flight rechecks) that no pass has run lately, a late-end scan that has
74+
* fallen behind, or a window reaching before a
6075
* backfill that is still running. Once the backfill is complete, instants before the
6176
* coverage hold no runs and are read live at the cost of an index probe per workspace.
6277
*/
@@ -70,6 +85,10 @@ export function planRollupRead(
7085
if (!coverageStart || !freshThrough) return null
7186
const lagging = (at: Date | null) => !at || now.getTime() - at.getTime() >= MAX_FRESHNESS_LAG_MS
7287
if (lagging(freshThrough) || lagging(state.correctionsAt)) return null
88+
const { lastEndScan } = state
89+
if (!lastEndScan || now.getTime() - lastEndScan.getTime() >= MAX_LATE_END_SCAN_LAG_MS) {
90+
return null
91+
}
7392
if (window.from < coverageStart && !state.backfillCompletedAt) return null
7493

7594
const from = new Date(Math.max(ceilHour(window.from).getTime(), coverageStart.getTime()))
@@ -84,7 +103,7 @@ export function planRollupRead(
84103
/**
85104
* Reads the state and the rows it vouches for in one snapshot of the replica, so a pass that
86105
* commits mid-request can never pair new progress with old cells. Null when the rollup cannot
87-
* serve the window.
106+
* serve the window, or when more of its cells are marked than are worth reading live.
88107
*/
89108
function readInSnapshot<T>(
90109
scope: ActivityScope,
@@ -97,7 +116,15 @@ function readInSnapshot<T>(
97116
{ from: scope.start, to: scope.end },
98117
new Date()
99118
)
100-
return plan ? read(tx, plan) : null
119+
if (!plan) return null
120+
if (plan.rollup && (await exceedsLiveMarkedCells(tx, scope, plan.rollup))) {
121+
logger.warn('Organization activity read too many marked cells; using the legacy read', {
122+
organizationId: scope.organizationId,
123+
maxLiveMarkedCells: MAX_LIVE_MARKED_CELLS,
124+
})
125+
return null
126+
}
127+
return read(tx, plan)
101128
},
102129
{ isolationLevel: 'repeatable read', accessMode: 'read only' }
103130
)
@@ -134,6 +161,22 @@ function markedCells(scope: ActivityScope, span: InstantRange): SQL {
134161
${scope.workspaceId ? sql`AND w.id = ${scope.workspaceId}` : sql``}`
135162
}
136163

164+
/**
165+
* Whether the scope has more than {@link MAX_LIVE_MARKED_CELLS} marked cells in `span`, counted
166+
* no further than one past the bound.
167+
*/
168+
async function exceedsLiveMarkedCells(
169+
tx: DbTransaction,
170+
scope: ActivityScope,
171+
span: InstantRange
172+
): Promise<boolean> {
173+
const [row] = await tx.execute<{ exceeds: boolean }>(sql`
174+
SELECT count(*) > ${MAX_LIVE_MARKED_CELLS} AS exceeds
175+
FROM (${markedCells(scope, span)} LIMIT ${MAX_LIVE_MARKED_CELLS + 1}) m
176+
`)
177+
return row?.exceeds === true
178+
}
179+
137180
/** A stored cell `r` of workspace `w` whose workspace-hour is not among `marked` cells `m`. */
138181
function notMarked(marked: SQL): SQL {
139182
return sql`NOT EXISTS (SELECT 1 FROM (${marked}) m WHERE m.workspace_id = w.id AND m.h = r.bucket_start)`

‎apps/sim/lib/billing/core/organization-activity-rollup.integration.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -923,6 +923,29 @@ describe('organization activity rollup', () => {
923923
expect((await readRollupState())?.failedPasses).toBe(0)
924924
}, 300_000)
925925

926+
it('counts a pass that died holding the lease as failed, so a deferred hour backs off', async () => {
927+
const hour = await reopenLastSettledHour()
928+
await connection.unsafe(
929+
`INSERT INTO activity_rollup_blocked VALUES ($1::timestamp, 0, now())`,
930+
[hour]
931+
)
932+
// Two passes threw; a third was killed paging the hour, so its lease was never released.
933+
await connection.unsafe(
934+
`UPDATE activity_rollup_state SET failed_passes = 2, lease_owner = 'killed-pass',
935+
lease_expires_at = now() - interval '1 second'`
936+
)
937+
await gateHour(hour, `RAISE EXCEPTION 'Synthetic crash';`)
938+
try {
939+
await expect(runPass()).rejects.toThrow()
940+
expect(await blockedAttempts(hour)).toBe(1)
941+
} finally {
942+
await ungateHour()
943+
}
944+
await connection.unsafe(`UPDATE activity_rollup_blocked SET retry_at = now()`)
945+
expect(await runPass()).toMatchObject({ finalizedHours: 1, blockedHours: 0 })
946+
expect((await readRollupState())?.failedPasses).toBe(0)
947+
}, 300_000)
948+
926949
it('commits a retry backoff before the retry, so a retry that dies still backs off', async () => {
927950
const hour = await reopenLastSettledHour()
928951
await connection.unsafe(
@@ -1119,6 +1142,41 @@ describe('organization activity rollup', () => {
11191142
await expectEquivalent()
11201143
}, 300_000)
11211144

1145+
it('stops serving the rollup while the late-end scan has fallen behind', async () => {
1146+
await connection.unsafe(
1147+
`UPDATE activity_rollup_state SET last_end_scan = (now() AT TIME ZONE 'UTC') - interval '1 hour'`
1148+
)
1149+
expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).toBeNull()
1150+
expect(
1151+
await readRollupActivityBreakdown(scopeOf(longWindow()), 'workflow', 'runs', 0)
1152+
).toBeNull()
1153+
await runPass()
1154+
expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).not.toBeNull()
1155+
await expectEquivalent()
1156+
}, 300_000)
1157+
1158+
it('uses the legacy read when more cells are marked than are worth reading live', async () => {
1159+
const state = await readRollupState()
1160+
if (!state?.freshThrough) throw new Error('No coverage')
1161+
// Past the read's bound of 5,000 marked cells: organization A's workspaces, inside the long
1162+
// window, due only after the test so no pass takes them.
1163+
await connection`INSERT INTO activity_rollup_dirty (workspace_id, bucket_start, due_at)
1164+
SELECT w, ${state.freshThrough.toISOString()}::timestamptz AT TIME ZONE 'UTC' - h * interval '1 hour',
1165+
now() + interval '1 day'
1166+
FROM unnest(ARRAY['ws1', 'ws2', 'ws4']) AS w, generate_series(1, 1700) AS h
1167+
ON CONFLICT (workspace_id, bucket_start) DO NOTHING`
1168+
try {
1169+
expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).toBeNull()
1170+
expect(
1171+
await readRollupActivityBreakdown(scopeOf(longWindow()), 'workflow', 'runs', 0)
1172+
).toBeNull()
1173+
} finally {
1174+
await connection`DELETE FROM activity_rollup_dirty WHERE due_at > now() + interval '1 hour'`
1175+
}
1176+
expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).not.toBeNull()
1177+
await expectEquivalent()
1178+
}, 300_000)
1179+
11221180
it('backs off an hour too heavy to read and keeps the rest of the pass running', async () => {
11231181
const [{ hour }] = await connection.unsafe<{ hour: string }[]>(
11241182
`SELECT to_char(h, 'YYYY-MM-DD HH24:MI:SS') AS hour FROM (

‎packages/db/schema.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -807,7 +807,10 @@ export const activityRollupState = pgTable('activity_rollup_state', {
807807
* rebuilds the rollup.
808808
*/
809809
maintainedAt: timestamp('maintained_at', { withTimezone: true }),
810-
/** Passes in a row that failed after taking the lease; reset by the next pass that completes. */
810+
/**
811+
* Passes in a row that failed after taking the lease, by throwing or by dying with it held;
812+
* reset by the next pass that completes.
813+
*/
811814
failedPasses: integer('failed_passes').notNull().default(0),
812815
/**
813816
* When a pass last ran both correction queues (dirty marks and in-flight rechecks) to the end of

0 commit comments

Comments
 (0)