From 1743a93147684b1b228b70f416c5e1c3df366215 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Mon, 16 Nov 2026 10:40:49 +0900 Subject: [PATCH] fix(usage): yield within indexed row decoding and aggregation Stream indexed rows into the private accumulator under the existing cooperative slice and honor cancellation within a candidate page. Revalidate fresh bytes and stable file identity after any yielding verification pass. Add regression coverage for scheduling, aborts, compatibility, and file mutations. --- .../management/usage-aggregate-cache.ts | 9 +- src/usage/ledger-scanner.ts | 54 ++++- structure/dashboard-and-usage.md | 2 +- structure/usage-client-attribution.md | 9 +- tests/usage/usage-aggregate-cache.test.ts | 38 +++ tests/usage/usage-ledger-scanner.test.ts | 216 +++++++++++++++++- 6 files changed, 311 insertions(+), 17 deletions(-) diff --git a/src/server/management/usage-aggregate-cache.ts b/src/server/management/usage-aggregate-cache.ts index 17e530fa866..6e7b02d0c2a 100644 --- a/src/server/management/usage-aggregate-cache.ts +++ b/src/server/management/usage-aggregate-cache.ts @@ -477,14 +477,18 @@ async function accumulateCandidates( try { let first = true; for (;;) { - // Candidate selection and row decoding are synchronous bounded chunks. A timer yield - // between them prevents a broad authenticated query from monopolizing the event loop. + // Candidate selection is synchronous and bounded. Row decoding and accumulation also + // yield within each page; keep the page-boundary yield for cheap rows and SQL work. if (!first) await new Promise(resolve => setTimeout(resolve, 0)); const offsets = read.nextPage(); if (!first && offsets.length === 0) break; first = false; const rows = await readUsageLedgerRowsAtOffsets({ offsets, + // This accumulator is private to the rebuild/append attempt and is discarded if the + // reader's final validation fails. Fold inside its time slice instead of retaining a + // page of parsed rows and then aggregating all of them in one synchronous turn. + onEntry: entry => accumulator.add(entry), expectedIdentityKey: read.snapshot.identityKey, // The offsets belong to this index snapshot. If the ledger moved since the snapshot, // the reader re-validates the boundary before trusting any of them, and a rewrite of @@ -499,7 +503,6 @@ async function accumulateCandidates( // Serving only the rows that happened to parse would silently under-report the filter. throw new UsageFilterIndexCorruptError(); } - for (const entry of rows.entries) accumulator.add(entry); if (offsets.length === 0) break; } } finally { diff --git a/src/usage/ledger-scanner.ts b/src/usage/ledger-scanner.ts index 2d469de105d..8738f2dfc2f 100644 --- a/src/usage/ledger-scanner.ts +++ b/src/usage/ledger-scanner.ts @@ -486,6 +486,12 @@ export async function scanUsageLedgerCooperatively( export interface ReadUsageLedgerRowsOptions { /** Ascending, distinct row-start offsets reported by an earlier scan of the same file. */ offsets: readonly number[]; + /** + * Folds rows as they are decoded instead of retaining them in `entries`. Callback work shares + * the cooperative time slice. Results are provisional until this read resolves: discard them + * if identity or content validation fails, as with the full-ledger scanner's callback. + */ + onEntry?: (entry: PersistedUsageEntry, rowStartBytes: number) => void; /** Identity those offsets were captured under; a replaced file invalidates all of them. */ expectedIdentityKey: string; /** Revision the offsets were captured under. Any movement re-validates the boundary. */ @@ -503,6 +509,7 @@ export interface ReadUsageLedgerRowsOptions { } export interface UsageLedgerRowRead { + /** Empty when an onEntry callback consumes the rows. */ entries: PersistedUsageEntry[]; /** Offsets that no longer frame a complete, normalizable row. */ unreadableOffsets: number[]; @@ -626,27 +633,32 @@ export async function readUsageLedgerRowsAtOffsets( return null; }; - const walk = (visit: (bytes: Buffer, offset: number) => boolean): number[] => { + const walk = async (visit: (bytes: Buffer, offset: number) => boolean): Promise => { const unreadableOffsets: number[] = []; + let lastYieldAt = performance.now(); for (const offset of options.offsets) { throwIfAborted(options.signal); if (!Number.isSafeInteger(offset) || offset < 0 || offset >= size) { unreadableOffsets.push(offset); - continue; + } else { + const bytes = lineAt(offset); + if (!bytes || !visit(bytes, offset)) unreadableOffsets.push(offset); } - const bytes = lineAt(offset); - if (!bytes) { - unreadableOffsets.push(offset); - continue; + // A bounded candidate count does not bound decoding or aggregation time. Include the + // visitor in the same slice as the full scanner, and let timers/cancellation run even + // within one page. A single row remains atomic. + if (performance.now() - lastYieldAt >= MAX_SCAN_SLICE_MS) { + await new Promise(resolve => setTimeout(resolve, 0)); + throwIfAborted(options.signal); + lastYieldAt = performance.now(); } - if (!visit(bytes, offset)) unreadableOffsets.push(offset); } return unreadableOffsets; }; const entries: PersistedUsageEntry[] = []; const captured = createHash("sha256"); - const unreadableOffsets = walk((bytes, offset) => { + const unreadableOffsets = await walk((bytes, offset) => { captured.update(bytes); captured.update("\0"); captured.update(String(offset)); @@ -660,7 +672,8 @@ export async function readUsageLedgerRowsAtOffsets( // normalizes means the ledger moved under the projection, so it is unreadable rather than // silently dropped: the caller rebuilds instead of reporting a smaller complete result. if (!entry) return false; - entries.push(entry); + if (options.onEntry) options.onEntry(entry, offset); + else entries.push(entry); return true; }); const capturedDigest = captured.digest("hex"); @@ -675,6 +688,7 @@ export async function readUsageLedgerRowsAtOffsets( if (!pathRevision || usageLogIdentityKey(pathRevision) !== options.expectedIdentityKey) { throw rebuildRequired("identity_mismatch"); } + if (pathRevision.size < size) throw rebuildRequired("shrink"); // Any write during the read can be an in-place rewrite, not just an append. Re-read the // same offsets and compare, so a rewrite that lands between two batches cannot publish a // mixture of old and new rows. A pure append leaves every selected row unchanged. @@ -686,8 +700,11 @@ export async function readUsageLedgerRowsAtOffsets( if (physical !== undefined && endingRevision.size <= physical) { throw rebuildRequired("content_changed"); } + // Verification must read the file again, not re-hash a small page still in the read cache. + windowStart = -1; + windowLength = 0; const verified = createHash("sha256"); - const recheck = walk((bytes, offset) => { + const recheck = await walk((bytes, offset) => { verified.update(bytes); verified.update("\0"); verified.update(String(offset)); @@ -697,7 +714,24 @@ export async function readUsageLedgerRowsAtOffsets( || verified.digest("hex") !== capturedDigest) { throw rebuildRequired("content_changed"); } + const verifiedFdRevision = revisionFromStat(path, fd, fstatSync(fd)); + const verifiedPathRevision = currentUsageLogRevision(); + if (!verifiedPathRevision + || usageLogIdentityKey(verifiedFdRevision) !== options.expectedIdentityKey + || usageLogIdentityKey(verifiedPathRevision) !== options.expectedIdentityKey) { + throw rebuildRequired("identity_mismatch"); + } + if (verifiedFdRevision.size < size || verifiedPathRevision.size < size) { + throw rebuildRequired("shrink"); + } + // Verification yields too. A later write could change bytes already checked, so retry + // rather than publish a result whose verification raced another mutation. + if (usageLogRevisionKey(verifiedFdRevision) !== usageLogRevisionKey(endingRevision) + || usageLogRevisionKey(verifiedPathRevision) !== usageLogRevisionKey(pathRevision)) { + throw rebuildRequired("content_changed"); + } } + throwIfAborted(options.signal); return { entries, unreadableOffsets }; } finally { closeSync(fd); diff --git a/structure/dashboard-and-usage.md b/structure/dashboard-and-usage.md index 5ca7d5ff2cf..55311980053 100644 --- a/structure/dashboard-and-usage.md +++ b/structure/dashboard-and-usage.md @@ -340,7 +340,7 @@ is treated as an append: the scanner verifies the previous LF and its trailing 6 folds only the suffix into a cloned accumulator and publishes it after validation. Concurrent callers share that work. Cold rebuilds and append scans read fixed-size chunks, yield between chunks after a full chunk or 1,000 rows, and yield between rows once 8 ms have passed, including within the final chunk, then recheck cancellation. This keeps costly normalization and -aggregation cooperative; one row remains atomic. The first read is proportional to ledger size; later refreshes hash the bounded retained window once and parse only the appended suffix, rather than rescanning the whole ledger. `tests/usage/usage-ledger-scanner.test.ts` covers slow-row scheduling and cancellation. +aggregation cooperative; one row remains atomic. Indexed candidate reads use the same time slice for decoding and direct accumulator callbacks within each page, including cooperative content verification before publication. The first read is proportional to ledger size; later refreshes hash the bounded retained window once and parse only the appended suffix, rather than rescanning the whole ledger. `tests/usage/usage-ledger-scanner.test.ts` covers slow-row scheduling, cancellation and validation across yields. The Dashboard polls its 30-day usage independently once per minute, separate from five-second state polls. An unchanged retained snapshot reuses its verified region digest only for identical bounds; appends or trimming hash the returned region, preserving same-inode rewrite detection. > Decision record: [ADR-0102](decisions/ADR-0102-incremental-stream-accounting.md) diff --git a/structure/usage-client-attribution.md b/structure/usage-client-attribution.md index 42da962df1c..8824856849c 100644 --- a/structure/usage-client-attribution.md +++ b/structure/usage-client-attribution.md @@ -42,12 +42,17 @@ row offset during ingest, and `readUsageLedgerRowsAtOffsets` re-reads only the s same identity, row-ceiling and torn-suffix rules. The first filtered query after the index is missing or invalid pays one cooperative full ingest; appends ingest only new bytes, and every later filter reuses the projection. Candidate offsets are selected and consumed in bounded keyset pages, with a -timer yield between pages; each page rechecks the published generation before it is read, so broad -filters neither retain the complete offset set nor monopolize the event loop. A replaced, truncated +timer yield between pages; each page rechecks the published generation before it is read. Within a +page, decoding and accumulation share the scanner's 8 ms cooperative slice, with one row atomic and +cancellation checked after yielding. The accumulator consumes rows directly instead of retaining a +page of parsed entries. A replaced, truncated or same-size rewritten ledger rebuilds it, a write observed during a row read re-verifies the selected bytes before publishing them, and a recorded offset that no longer frames a complete row drops the projection instead of serving a partial answer. `usage.jsonl` stays authoritative and is never truncated or rewritten. +Row callbacks are provisional until validation completes; failed rebuilds discard their accumulator +and failed appends discard their clone. A verification pass clears the cached read window, yields +between rows, and rejects identity, size or revision changes during that pass before publication. Each candidate page is one keyset statement per branch, and every branch index is ordered `(generation, , byte_offset, ...)` so its own `ORDER BY byte_offset LIMIT` is diff --git a/tests/usage/usage-aggregate-cache.test.ts b/tests/usage/usage-aggregate-cache.test.ts index 448829e52c1..91d201d1961 100644 --- a/tests/usage/usage-aggregate-cache.test.ts +++ b/tests/usage/usage-aggregate-cache.test.ts @@ -22,6 +22,7 @@ import { import type { OcxConfig } from "../../src/types/config"; import { resetUsageReadCacheForTests, type PersistedUsageEntry } from "../../src/usage/log"; import * as usageLedgerScannerModule from "../../src/usage/ledger-scanner"; +import * as usageSummaryModule from "../../src/usage/summary"; import { getUsageSourceFacets, SourceFacetsBusyError } from "../../src/server/management/usage-source-facets"; import { refreshUserCostOverlays } from "../../src/usage/user-cost-overlays"; import { buildRouteDecisionTrace } from "../../src/routing/trace"; @@ -81,6 +82,43 @@ afterEach(() => { }); describe("retained usage aggregate cache", () => { + test("costly accumulator work yields within one indexed candidate page", async () => { + const count = 40; + writeFileSync(join(testDir, "usage.jsonl"), Array.from({ length: count }, (_, index) => line(`costly-${index}`)).join("")); + // Build the shared index with another filter so ingestion cannot supply the observed yield. + const warm = await getFilteredUsageAggregate({ provider: "anthropic" }); + expect(warm.filterIndex).toMatchObject({ totalRows: count, candidateRows: 0 }); + let elapsed = 0; + let visited = 0; + let visitedAtTimer = -1; + let timer: ReturnType | undefined; + const original = usageSummaryModule.createUsageSummaryAccumulator; + const clock = spyOn(performance, "now").mockImplementation(() => elapsed); + const create = spyOn(usageSummaryModule, "createUsageSummaryAccumulator").mockImplementation(options => { + const accumulator = original(options); + const add = accumulator.add.bind(accumulator); + accumulator.add = value => { + add(value); + visited += 1; + elapsed += 10; + if (timer === undefined) timer = setTimeout(() => { visitedAtTimer = visited; }, 0); + }; + return accumulator; + }); + try { + const result = await getFilteredUsageAggregate({ provider: "openai" }); + expect(result.filterIndex).toMatchObject({ totalRows: count, candidateRows: count }); + expect(visited).toBe(count); + expect(visitedAtTimer).toBeGreaterThan(0); + expect(visitedAtTimer).toBeLessThan(count); + expect(result.accumulator.summarize("all", NOW).summary).toMatchObject({ requests: count, totalTokens: count * 2 }); + } finally { + if (timer !== undefined) clearTimeout(timer); + create.mockRestore(); + clock.mockRestore(); + } + }); + test("app breakdowns survive filtered cache appends and match fresh scans", async () => { const path = join(testDir, "usage.jsonl"); const row = (id: string, client: "codex_cli" | "claude_code", apiKeyId: string) => ({ ...entry(id), client, apiKeyId }); diff --git a/tests/usage/usage-ledger-scanner.test.ts b/tests/usage/usage-ledger-scanner.test.ts index e6af1574fc4..6d27e10d3b2 100644 --- a/tests/usage/usage-ledger-scanner.test.ts +++ b/tests/usage/usage-ledger-scanner.test.ts @@ -1,15 +1,18 @@ import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { createHash } from "node:crypto"; import { appendFileSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import * as fs from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { + readUsageLedgerRowsAtOffsets, scanUsageLedgerCooperatively, USAGE_LEDGER_BOUNDARY_DIGEST_BYTES, USAGE_LEDGER_MAX_LINE_BYTES, UsageLedgerRebuildRequiredError, + type ReadUsageLedgerRowsOptions, } from "../../src/usage/ledger-scanner"; -import { normalizePersistedUsageRow, usageLogIdentityKey, usageLogPath, type PersistedUsageEntry } from "../../src/usage/log"; +import { currentUsageLogRevision, normalizePersistedUsageRow, usageLogIdentityKey, usageLogPath, usageLogRevisionKey, type PersistedUsageEntry } from "../../src/usage/log"; let testDir = ""; let previousHome: string | undefined; @@ -33,6 +36,30 @@ function line(requestId: string, overrides: Partial = {}): return `${JSON.stringify(entry(requestId, overrides))}\n`; } +function indexedFixture(count = 40): { options: ReadUsageLedgerRowsOptions; contents: string; ids: string[] } { + const ids = Array.from({ length: count }, (_, index) => `indexed-${index}`); + const rows = ids.map(id => line(id)); + const offsets: number[] = []; + let size = 0; + for (const row of rows) { offsets.push(size); size += Buffer.byteLength(row); } + const contents = rows.join(""); + writeFileSync(usageLogPath(), contents); + const revision = currentUsageLogRevision(); + return { + contents, + ids, + options: { + offsets, + expectedIdentityKey: usageLogIdentityKey(revision), + expectedRevisionKey: usageLogRevisionKey(revision), + expectedProcessedThroughBytes: size, + expectedProcessedThroughDigest: createHash("sha256") + .update(Buffer.from(contents).subarray(-USAGE_LEDGER_BOUNDARY_DIGEST_BYTES)).digest("hex"), + expectedFileSize: size, + }, + }; +} + beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; testDir = mkdtempSync(join(tmpdir(), "ocx-usage-ledger-scan-")); @@ -592,3 +619,190 @@ describe("usage ledger cooperative scanner", () => { })).rejects.toBe(sentinel); }); }); + +describe("usage ledger indexed row reader", () => { + test("folds a single page cooperatively without retaining entries", async () => { + const { options, ids, contents } = indexedFixture(); + expect(Buffer.byteLength(contents)).toBeLessThan(USAGE_LEDGER_BOUNDARY_DIGEST_BYTES); + const seen: Array<[string, number]> = []; + let elapsed = 0; + let visitedAtTimer = -1; + const clock = spyOn(performance, "now").mockImplementation(() => elapsed); + const timer = setTimeout(() => { visitedAtTimer = seen.length; }, 0); + try { + const result = await readUsageLedgerRowsAtOffsets({ + ...options, + onEntry(value, offset) { seen.push([value.requestId, offset]); elapsed += 10; }, + }); + expect(visitedAtTimer).toBeGreaterThan(0); + expect(visitedAtTimer).toBeLessThan(ids.length); + expect(seen).toEqual(ids.map((id, index) => [id, options.offsets[index]])); + expect(result).toEqual({ entries: [], unreadableOffsets: [] }); + } finally { + clearTimeout(timer); + clock.mockRestore(); + } + }); + + test("cancels inside one page with the original reason and closes its descriptor", async () => { + const { options, ids } = indexedFixture(); + const controller = new AbortController(); + const reason = new Error("cancel indexed page"); + let visited = 0; + let elapsed = 0; + const clock = spyOn(performance, "now").mockImplementation(() => elapsed); + const opened = spyOn(fs, "openSync"); + const closed = spyOn(fs, "closeSync"); + const timer = setTimeout(() => controller.abort(reason), 0); + try { + await expect(readUsageLedgerRowsAtOffsets({ + ...options, + signal: controller.signal, + onEntry() { visited += 1; elapsed += 10; }, + })).rejects.toBe(reason); + expect(visited).toBeGreaterThan(0); + expect(visited).toBeLessThan(ids.length); + const fd = opened.mock.results[0]?.value as number; + expect(typeof fd).toBe("number"); + expect(closed).toHaveBeenCalledWith(fd); + expect(() => fs.fstatSync(fd)).toThrow(); + } finally { + clearTimeout(timer); + closed.mockRestore(); + opened.mockRestore(); + clock.mockRestore(); + } + }); + + test("propagates a fold failure instead of treating the row as unreadable", async () => { + const { options } = indexedFixture(2); + const reason = new Error("indexed fold failed"); + let visited = 0; + await expect(readUsageLedgerRowsAtOffsets({ + ...options, + onEntry() { visited += 1; throw reason; }, + })).rejects.toBe(reason); + expect(visited).toBe(1); + }); + + test("keeps the returned-array contract and selected row order without a fold callback", async () => { + const { options, ids } = indexedFixture(5); + const selected = [0, 2, 4]; + const result = await readUsageLedgerRowsAtOffsets({ + ...options, + offsets: selected.map(index => options.offsets[index]!), + }); + expect(result.entries).toEqual(selected.map(index => entry(ids[index]!))); + expect(result.unreadableOffsets).toEqual([]); + }); + + test("rejects a rewrite plus append even when every selected row fits the cached read window", async () => { + const { options, contents } = indexedFixture(); + expect(Buffer.byteLength(contents)).toBeLessThan(USAGE_LEDGER_BOUNDARY_DIGEST_BYTES); + let elapsed = 0; + let rewritten = false; + const clock = spyOn(performance, "now").mockImplementation(() => elapsed); + const timer = setTimeout(() => { + writeFileSync(usageLogPath(), contents.replace("indexed-0", "changed-0") + line("appended")); + rewritten = true; + }, 0); + try { + await expect(readUsageLedgerRowsAtOffsets({ + ...options, + onEntry() { elapsed += 10; }, + })).rejects.toMatchObject({ code: "usage_ledger_rebuild_required", reason: "content_changed" }); + expect(rewritten).toBe(true); + } finally { + clearTimeout(timer); + clock.mockRestore(); + } + }); + + test("accepts a pure append while folding exactly the original selection once", async () => { + const { options, ids } = indexedFixture(); + const seen: string[] = []; + let elapsed = 0; + let appended = false; + const clock = spyOn(performance, "now").mockImplementation(() => elapsed); + const timer = setTimeout(() => { appendFileSync(usageLogPath(), line("appended")); appended = true; }, 0); + try { + const result = await readUsageLedgerRowsAtOffsets({ + ...options, + onEntry(value) { seen.push(value.requestId); elapsed += 10; }, + }); + expect(appended).toBe(true); + expect(seen).toEqual(ids); + expect(result).toEqual({ entries: [], unreadableOffsets: [] }); + } finally { + clearTimeout(timer); + clock.mockRestore(); + } + }); + + test.each(["replacement", "shrink", "rewrite-append", "abort"] as const)( + "refuses %s delivered during cooperative verification", + async kind => { + const { options, contents, ids } = indexedFixture(); + const controller = new AbortController(); + const reason = new Error("cancel indexed verification"); + let visited = 0; + let elapsed = 0; + let mutationRan = false; + let timer: ReturnType | undefined; + const clock = spyOn(performance, "now").mockImplementation(() => elapsed); + const realRead = fs.readSync; + const reads = spyOn(fs, "readSync").mockImplementation((...args: Parameters) => { + const result = realRead(...args); + // Arm the race only after every row was folded and verification reads fresh bytes. + if (visited === ids.length && timer === undefined) { + elapsed += 10; + timer = setTimeout(() => { + if (kind === "replacement") { + renameSync(usageLogPath(), `${usageLogPath()}.old`); + writeFileSync(usageLogPath(), contents); + } else if (kind === "shrink") { + writeFileSync(usageLogPath(), line("short")); + } else if (kind === "rewrite-append") { + writeFileSync(usageLogPath(), contents.replace("indexed-0", "changed-0") + line("appended-again")); + } else controller.abort(reason); + mutationRan = true; + }, 0); + } + return result; + }); + try { + const read = readUsageLedgerRowsAtOffsets({ + ...options, + signal: controller.signal, + onEntry() { + visited += 1; + // Force a verification pass while leaving every selected row unchanged. + if (visited === ids.length) appendFileSync(usageLogPath(), line("appended")); + }, + }); + if (kind === "abort") await expect(read).rejects.toBe(reason); + else await expect(read).rejects.toMatchObject({ + code: "usage_ledger_rebuild_required", + reason: kind === "replacement" ? "identity_mismatch" : kind === "shrink" ? "shrink" : "content_changed", + }); + expect(mutationRan).toBe(true); + expect(visited).toBe(ids.length); + } finally { + if (timer !== undefined) clearTimeout(timer); + reads.mockRestore(); + clock.mockRestore(); + } + }, + ); + + test("honors cancellation from the final callback without needing another row or timer", async () => { + const { options } = indexedFixture(1); + const controller = new AbortController(); + const reason = new Error("cancel after final indexed row"); + await expect(readUsageLedgerRowsAtOffsets({ + ...options, + signal: controller.signal, + onEntry() { controller.abort(reason); }, + })).rejects.toBe(reason); + }); +});