Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 26 additions & 7 deletions apps/sim/lib/billing/activity-rollup/pass.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import {
resetRollup,
truncateRollupBatch,
updateLeasedState,
WORKFLOW_ROLLUP_ID,
} from '@/lib/billing/activity-rollup/state'
import type { DbTransaction } from '@/lib/db/types'

Expand Down Expand Up @@ -143,8 +144,9 @@ function startCoverage(
}

/**
* Once this many passes in a row have failed, a deferred hour counts as a retry, so a pass that
* keeps crashing while paging it backs it off instead of starting it again every tick.
* Once this many passes in a row have failed (thrown, or died holding the lease), a deferred hour
* counts as a retry, so a pass that keeps crashing while paging it backs it off instead of starting
* it again every tick.
*/
const CRASHING_PASSES = 3

Expand Down Expand Up @@ -663,14 +665,31 @@ export async function runActivityRollupPass(
}
}

/** Counts a pass that failed after taking the lease, and logs the run of failures loudly. */
/**
* Counts a pass that failed after taking the lease and releases the lease in the same update, so a
* takeover never counts the failure again; logs the run of failures loudly. Matched on the owner
* alone, so a pass that overran its lease is still counted; one whose lease another pass took over
* was already counted by that takeover, and matches nothing.
*/
async function recordFailedPass(owner: string, error: unknown): Promise<void> {
try {
const state = await updateLeasedState(owner, {
failedPasses: sql`${activityRollupState.failedPasses} + 1`,
})
const [state] = await db
.update(activityRollupState)
.set({
failedPasses: sql`${activityRollupState.failedPasses} + 1`,
leaseOwner: null,
leaseExpiresAt: null,
updatedAt: sql`now()`,
})
.where(
and(
eq(activityRollupState.id, WORKFLOW_ROLLUP_ID),
eq(activityRollupState.leaseOwner, owner)
)
)
.returning({ failedPasses: activityRollupState.failedPasses })
logger.error('Activity rollup pass failed', {
consecutiveFailures: state.failedPasses,
consecutiveFailures: state?.failedPasses ?? null,
error: getErrorMessage(error),
})
} catch (countError) {
Expand Down
6 changes: 6 additions & 0 deletions apps/sim/lib/billing/activity-rollup/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ export async function readRollupState(
* Takes the pass lease, creating the state row on first use, so two passes never run at once:
* Trigger.dev runs one at a time, but an inline pass and a Trigger.dev pass can meet while a
* deployment changes how it runs them. Returns the state, or null when another pass holds it.
*
* Every pass releases the lease on its way out, so one taken over after it expired belonged to a
* pass that died holding it (killed, or out of memory) and is counted as a failed pass, like one
* that threw. A lease left behind for any other reason (its release failing) counts the same way,
* which at worst backs off a deferred hour one failure early.
*/
export async function acquireLease(owner: string): Promise<ActivityRollupState | null> {
await db
Expand All @@ -63,6 +68,7 @@ export async function acquireLease(owner: string): Promise<ActivityRollupState |
.set({
leaseOwner: owner,
leaseExpiresAt: sql`now() + ${msInterval(LEASE_MS)}`,
failedPasses: sql`${activityRollupState.failedPasses} + CASE WHEN ${activityRollupState.leaseOwner} IS NULL THEN 0 ELSE 1 END`,
updatedAt: sql`now()`,
})
.where(
Expand Down
54 changes: 49 additions & 5 deletions apps/sim/lib/billing/core/organization-activity-rollup-queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,20 @@ const logger = createLogger('OrganizationActivityRollupQueries')
*/
const MAX_FRESHNESS_LAG_MS = 6 * HOUR_MS

/**
* Reads use the rollup only while the late-end scan is this recent: a run that ends late leaves
* its stored cell stale until the scan marks it. The scan trails now by its margin plus the time
* between passes (longer while a heavy-hour retry holds the lease), a quarter hour at most, so
* this bound is only crossed when the scan is failing or catching up a backlog.
*/
const MAX_LATE_END_SCAN_LAG_MS = HOUR_MS / 2

/**
* Each marked cell is read live as its own workspace-hour; past this many in a window, those
* reads approach the cost of the legacy read itself, so the legacy read serves instead.
*/
const MAX_LIVE_MARKED_CELLS = 5_000

/** Which part of a window the rollup serves, and the instants read live from the source. */
export interface RollupReadPlan {
/** Whole UTC hours, all computed. */
Expand All @@ -56,7 +70,8 @@ export interface RollupReadPlan {
/**
* Splits `window` between the rollup and the source, or returns null when the rollup must not
* serve it at all: another version's cells, no coverage yet, a stalled pass, correction queues
* (dirty marks and in-flight rechecks) that no pass has run lately, or a window reaching before a
* (dirty marks and in-flight rechecks) that no pass has run lately, a late-end scan that has
* fallen behind, or a window reaching before a
* backfill that is still running. Once the backfill is complete, instants before the
* coverage hold no runs and are read live at the cost of an index probe per workspace.
*/
Expand All @@ -70,6 +85,10 @@ export function planRollupRead(
if (!coverageStart || !freshThrough) return null
const lagging = (at: Date | null) => !at || now.getTime() - at.getTime() >= MAX_FRESHNESS_LAG_MS
if (lagging(freshThrough) || lagging(state.correctionsAt)) return null
const { lastEndScan } = state
if (!lastEndScan || now.getTime() - lastEndScan.getTime() >= MAX_LATE_END_SCAN_LAG_MS) {
return null
}
if (window.from < coverageStart && !state.backfillCompletedAt) return null

const from = new Date(Math.max(ceilHour(window.from).getTime(), coverageStart.getTime()))
Expand All @@ -84,7 +103,7 @@ export function planRollupRead(
/**
* Reads the state and the rows it vouches for in one snapshot of the replica, so a pass that
* commits mid-request can never pair new progress with old cells. Null when the rollup cannot
* serve the window.
* serve the window, or when more of its cells are marked than are worth reading live.
*/
function readInSnapshot<T>(
scope: ActivityScope,
Expand All @@ -97,7 +116,15 @@ function readInSnapshot<T>(
{ from: scope.start, to: scope.end },
new Date()
)
return plan ? read(tx, plan) : null
if (!plan) return null
if (plan.rollup && (await exceedsLiveMarkedCells(tx, scope, plan.rollup))) {
logger.info('Organization activity read too many marked cells; using the legacy read', {
organizationId: scope.organizationId,
maxLiveMarkedCells: MAX_LIVE_MARKED_CELLS,
})
return null
}
return read(tx, plan)
},
{ isolationLevel: 'repeatable read', accessMode: 'read only' }
)
Expand All @@ -124,16 +151,33 @@ function storedCells(scope: ActivityScope, span: InstantRange): SQL {
* The rollup cells of the scope's workspaces that have a pending dirty mark, as
* `(workspace_id, h)`: known to be stale until the mark is processed (or while it backs off after
* failing), so reads take exactly those workspace-hours live and every other workspace's stored
* cell for the hour still serves. Read on the mark queue's key, in the read's snapshot.
* cell for the hour still serves. Read on the mark queue's key, in the read's snapshot. Each row is
* a distinct cell: the queue's key is the workspace-hour, and each mark joins one workspace.
*/
function markedCells(scope: ActivityScope, span: InstantRange): SQL {
return sql`SELECT DISTINCT d.workspace_id, d.bucket_start AS h FROM ${workspace} w
return sql`SELECT d.workspace_id, d.bucket_start AS h FROM ${workspace} w
JOIN ${activityRollupDirty} d ON d.workspace_id = w.id
AND d.bucket_start >= ${utcTimestamp(span.from)} AND d.bucket_start < ${utcTimestamp(span.to)}
WHERE w.organization_id = ${scope.organizationId}
${scope.workspaceId ? sql`AND w.id = ${scope.workspaceId}` : sql``}`
}

/**
* Whether the scope has more than {@link MAX_LIVE_MARKED_CELLS} marked cells in `span`, reading
* no further than one past the bound.
*/
async function exceedsLiveMarkedCells(
tx: DbTransaction,
scope: ActivityScope,
span: InstantRange
): Promise<boolean> {
const [row] = await tx.execute<{ exceeds: boolean }>(sql`
SELECT count(*) > ${MAX_LIVE_MARKED_CELLS} AS exceeds
FROM (${markedCells(scope, span)} LIMIT ${MAX_LIVE_MARKED_CELLS + 1}) m
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
`)
return row?.exceeds === true
}

/** A stored cell `r` of workspace `w` whose workspace-hour is not among `marked` cells `m`. */
function notMarked(marked: SQL): SQL {
return sql`NOT EXISTS (SELECT 1 FROM (${marked}) m WHERE m.workspace_id = w.id AND m.h = r.bucket_start)`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -923,6 +923,29 @@ describe('organization activity rollup', () => {
expect((await readRollupState())?.failedPasses).toBe(0)
}, 300_000)

it('counts a pass that died holding the lease as failed, so a deferred hour backs off', async () => {
const hour = await reopenLastSettledHour()
await connection.unsafe(
`INSERT INTO activity_rollup_blocked VALUES ($1::timestamp, 0, now())`,
[hour]
)
// Two passes threw; a third was killed paging the hour, so its lease was never released.
await connection.unsafe(
`UPDATE activity_rollup_state SET failed_passes = 2, lease_owner = 'killed-pass',
lease_expires_at = now() - interval '1 second'`
)
await gateHour(hour, `RAISE EXCEPTION 'Synthetic crash';`)
try {
await expect(runPass()).rejects.toThrow()
expect(await blockedAttempts(hour)).toBe(1)
} finally {
await ungateHour()
}
await connection.unsafe(`UPDATE activity_rollup_blocked SET retry_at = now()`)
expect(await runPass()).toMatchObject({ finalizedHours: 1, blockedHours: 0 })
expect((await readRollupState())?.failedPasses).toBe(0)
}, 300_000)

it('commits a retry backoff before the retry, so a retry that dies still backs off', async () => {
const hour = await reopenLastSettledHour()
await connection.unsafe(
Expand Down Expand Up @@ -1119,6 +1142,41 @@ describe('organization activity rollup', () => {
await expectEquivalent()
}, 300_000)

it('stops serving the rollup while the late-end scan has fallen behind', async () => {
await connection.unsafe(
`UPDATE activity_rollup_state SET last_end_scan = (now() AT TIME ZONE 'UTC') - interval '1 hour'`
)
expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).toBeNull()
expect(
await readRollupActivityBreakdown(scopeOf(longWindow()), 'workflow', 'runs', 0)
).toBeNull()
await runPass()
expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).not.toBeNull()
await expectEquivalent()
}, 300_000)

it('uses the legacy read when more cells are marked than are worth reading live', async () => {
const state = await readRollupState()
if (!state?.freshThrough) throw new Error('No coverage')
// Past the read's bound of 5,000 marked cells: organization A's workspaces, inside the long
// window, due only after the test so no pass takes them.
await connection`INSERT INTO activity_rollup_dirty (workspace_id, bucket_start, due_at)
SELECT w, ${state.freshThrough.toISOString()}::timestamptz AT TIME ZONE 'UTC' - h * interval '1 hour',
now() + interval '1 day'
FROM unnest(ARRAY['ws1', 'ws2', 'ws4']) AS w, generate_series(1, 1700) AS h
ON CONFLICT (workspace_id, bucket_start) DO NOTHING`
try {
expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).toBeNull()
expect(
await readRollupActivityBreakdown(scopeOf(longWindow()), 'workflow', 'runs', 0)
).toBeNull()
} finally {
await connection`DELETE FROM activity_rollup_dirty WHERE due_at > now() + interval '1 hour'`
}
expect(await readRollupActivityDays(scopeOf(longWindow()), 'UTC')).not.toBeNull()
await expectEquivalent()
}, 300_000)

it('backs off an hour too heavy to read and keeps the rest of the pass running', async () => {
const [{ hour }] = await connection.unsafe<{ hour: string }[]>(
`SELECT to_char(h, 'YYYY-MM-DD HH24:MI:SS') AS hour FROM (
Expand Down
5 changes: 4 additions & 1 deletion packages/db/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -807,7 +807,10 @@ export const activityRollupState = pgTable('activity_rollup_state', {
* rebuilds the rollup.
*/
maintainedAt: timestamp('maintained_at', { withTimezone: true }),
/** Passes in a row that failed after taking the lease; reset by the next pass that completes. */
/**
* Passes in a row that failed after taking the lease, by throwing or by dying with it held;
* reset by the next pass that completes.
*/
failedPasses: integer('failed_passes').notNull().default(0),
/**
* When a pass last ran both correction queues (dirty marks and in-flight rechecks) to the end of
Expand Down
Loading