@@ -45,6 +45,20 @@ const logger = createLogger('OrganizationActivityRollupQueries')
4545 */
4646const MAX_FRESHNESS_LAG_MS = 6 * HOUR_MS
4747
48+ /**
49+ * Reads use the rollup only while the late-end scan is this recent: a run that ends late leaves
50+ * its stored cell stale until the scan marks it. The scan trails now by its margin plus the time
51+ * between passes, minutes in all, so this bound is only crossed when the scan is failing or
52+ * catching up a backlog.
53+ */
54+ const MAX_LATE_END_SCAN_LAG_MS = HOUR_MS / 2
55+
56+ /**
57+ * Each marked cell is read live as its own workspace-hour; past this many in a window, those
58+ * reads approach the cost of the legacy read itself, so the legacy read serves instead.
59+ */
60+ const MAX_LIVE_MARKED_CELLS = 5_000
61+
4862/** Which part of a window the rollup serves, and the instants read live from the source. */
4963export interface RollupReadPlan {
5064 /** Whole UTC hours, all computed. */
@@ -56,7 +70,8 @@ export interface RollupReadPlan {
5670/**
5771 * Splits `window` between the rollup and the source, or returns null when the rollup must not
5872 * serve it at all: another version's cells, no coverage yet, a stalled pass, correction queues
59- * (dirty marks and in-flight rechecks) that no pass has run lately, or a window reaching before a
73+ * (dirty marks and in-flight rechecks) that no pass has run lately, a late-end scan that has
74+ * fallen behind, or a window reaching before a
6075 * backfill that is still running. Once the backfill is complete, instants before the
6176 * coverage hold no runs and are read live at the cost of an index probe per workspace.
6277 */
@@ -70,6 +85,10 @@ export function planRollupRead(
7085 if ( ! coverageStart || ! freshThrough ) return null
7186 const lagging = ( at : Date | null ) => ! at || now . getTime ( ) - at . getTime ( ) >= MAX_FRESHNESS_LAG_MS
7287 if ( lagging ( freshThrough ) || lagging ( state . correctionsAt ) ) return null
88+ const { lastEndScan } = state
89+ if ( ! lastEndScan || now . getTime ( ) - lastEndScan . getTime ( ) >= MAX_LATE_END_SCAN_LAG_MS ) {
90+ return null
91+ }
7392 if ( window . from < coverageStart && ! state . backfillCompletedAt ) return null
7493
7594 const from = new Date ( Math . max ( ceilHour ( window . from ) . getTime ( ) , coverageStart . getTime ( ) ) )
@@ -84,7 +103,7 @@ export function planRollupRead(
84103/**
85104 * Reads the state and the rows it vouches for in one snapshot of the replica, so a pass that
86105 * commits mid-request can never pair new progress with old cells. Null when the rollup cannot
87- * serve the window.
106+ * serve the window, or when more of its cells are marked than are worth reading live .
88107 */
89108function readInSnapshot < T > (
90109 scope : ActivityScope ,
@@ -97,7 +116,15 @@ function readInSnapshot<T>(
97116 { from : scope . start , to : scope . end } ,
98117 new Date ( )
99118 )
100- return plan ? read ( tx , plan ) : null
119+ if ( ! plan ) return null
120+ if ( plan . rollup && ( await exceedsLiveMarkedCells ( tx , scope , plan . rollup ) ) ) {
121+ logger . warn ( 'Organization activity read too many marked cells; using the legacy read' , {
122+ organizationId : scope . organizationId ,
123+ maxLiveMarkedCells : MAX_LIVE_MARKED_CELLS ,
124+ } )
125+ return null
126+ }
127+ return read ( tx , plan )
101128 } ,
102129 { isolationLevel : 'repeatable read' , accessMode : 'read only' }
103130 )
@@ -134,6 +161,22 @@ function markedCells(scope: ActivityScope, span: InstantRange): SQL {
134161 ${ scope . workspaceId ? sql `AND w.id = ${ scope . workspaceId } ` : sql `` } `
135162}
136163
164+ /**
165+ * Whether the scope has more than {@link MAX_LIVE_MARKED_CELLS} marked cells in `span`, counted
166+ * no further than one past the bound.
167+ */
168+ async function exceedsLiveMarkedCells (
169+ tx : DbTransaction ,
170+ scope : ActivityScope ,
171+ span : InstantRange
172+ ) : Promise < boolean > {
173+ const [ row ] = await tx . execute < { exceeds : boolean } > ( sql `
174+ SELECT count(*) > ${ MAX_LIVE_MARKED_CELLS } AS exceeds
175+ FROM (${ markedCells ( scope , span ) } LIMIT ${ MAX_LIVE_MARKED_CELLS + 1 } ) m
176+ ` )
177+ return row ?. exceeds === true
178+ }
179+
137180/** A stored cell `r` of workspace `w` whose workspace-hour is not among `marked` cells `m`. */
138181function notMarked ( marked : SQL ) : SQL {
139182 return sql `NOT EXISTS (SELECT 1 FROM (${ marked } ) m WHERE m.workspace_id = w.id AND m.h = r.bucket_start)`
0 commit comments