diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 5dbb28454e9..661da23a92d 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1810,7 +1810,8 @@ "zz-ci-storage-policy-isolation.test.ts": "ci-workflows", "zz-pr-coderabbit-readiness-revalidation.test.ts": "ci-workflows", "upstream-refusal-log.test.ts": "lib", - "call-usage-line-log.test.ts": "lib" + "call-usage-line-log.test.ts": "lib", + "request-summary-line.test.ts": "lib" }, "migrated": [ "adapters", diff --git a/src/lib/usage-line-log.ts b/src/lib/usage-line-log.ts index a23165ae5fc..61d42d6c840 100644 --- a/src/lib/usage-line-log.ts +++ b/src/lib/usage-line-log.ts @@ -1,4 +1,4 @@ -import { createHash } from "node:crypto"; +import { createHash, createHmac, randomBytes } from "node:crypto"; import { baseProviderLabel } from "../providers/label"; import { sanitizeLogMetadataString } from "./redact"; @@ -7,9 +7,11 @@ import { sanitizeLogMetadataString } from "./redact"; * the share of calls and tokens on each model from the process log alone. * * The line carries only the HTTP status, the provider and model ids, three token counts, an 8-hex - * hash of the gateway API key's record id and the duration. It never carries message text, a - * request or response body, a header, a secret key value, an account label or an email: the input is - * the fork's own final request record, and only the fields named below are read from it. + * hash of the gateway API key's record id, the duration, the time to first token, an 8-hex keyed tag of + * the serving account's label and three 0/1 flags (a 429 was seen, a retry happened, the request + * moved to another account or combo target). It never carries message text, a request or response + * body, a header, a secret key value, an account label in clear or an email: the input is the + * fork's own final request record, and only the fields named below are read from it. * * It is written only on an AgentRadar gateway (`OPENCODEX_AGENTRADAR=1`, the condition the * `[ar-acct]` line uses), so a self-hosted gateway gains no new output. @@ -32,6 +34,21 @@ export interface UsageLogInput { /** The configured key's record id (never the secret key value). Absent for environment or loopback admission. */ apiKeyId?: string; durationMs?: number; + /** Ms from request start to the first non-empty output delta; absent for non-streaming or tool-only calls. */ + firstOutputMs?: number; + /** The serving account's stable non-PII label; only its 8-hex hash is logged. */ + accountLogLabel?: string; + /** True only when combo execution advanced to another configured target. */ + comboTargetAdvanced?: boolean; + /** Upstream sends for the whole logical request; more than one means a retry. */ + spend?: { sends?: number }; + /** One row per physical attempt; only status, recovery counts and the account label are read. */ + attempts?: readonly { + status?: number; + recoveryKinds?: readonly unknown[]; + recoveryCount?: number; + accountLogLabel?: string; + }[]; } /** The OAuth (`o`) and API-key (`k`) account labels `baseProviderLabel` leaves in place. */ @@ -54,20 +71,53 @@ function count(value: number | undefined): string { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? String(Math.round(value)) : "-"; } +/** First 8 hex of sha256 of a non-empty id, or "-" when there is none. */ +function hash8(id: string | undefined): string { + if (typeof id !== "string" || id === "") return "-"; + return createHash("sha256").update(id).digest("hex").slice(0, 8); +} + /** First 8 hex of sha256 of the key record id, or "-" when the call carried no configured key. */ export function hashedKeyId(apiKeyId: string | undefined): string { - if (typeof apiKeyId !== "string" || apiKeyId === "") return "-"; - return createHash("sha256").update(apiKeyId).digest("hex").slice(0, 8); + return hash8(apiKeyId); +} + +/** + * Account labels are guessable (a provider name plus a short hex suffix), so a plain short hash could be + * reversed by trying them all. The tag is an HMAC under a random key made once per process and never + * written anywhere: the same account gets the same tag for the life of the process, and nothing more. + */ +const ACCOUNT_TAG_KEY = randomBytes(32); + +/** First 8 hex of the keyed tag of the serving account label (the entry's, else the last attempt's), or "-". */ +export function hashedAccount(input: Pick): string { + const last = [...(input.attempts ?? [])].reverse().find(a => typeof a.accountLogLabel === "string" && a.accountLogLabel !== ""); + const label = input.accountLogLabel || last?.accountLogLabel; + if (typeof label !== "string" || label === "") return "-"; + return createHmac("sha256", ACCOUNT_TAG_KEY).update(label).digest("hex").slice(0, 8); +} + +/** 429 / retry / fallover as 0 or 1, read from the attempt rows, spend total and combo flag only. */ +export function usageLogFlags(input: UsageLogInput): { r429: 0 | 1; retry: 0 | 1; fallover: 0 | 1 } { + const attempts = input.attempts ?? []; + const labels = new Set(attempts.map(a => a.accountLogLabel).filter((l): l is string => typeof l === "string" && l !== "")); + return { + r429: input.status === 429 || attempts.some(a => a.status === 429) ? 1 : 0, + retry: (input.spend?.sends ?? 0) > 1 || attempts.some(a => (a.recoveryCount ?? 0) > 0 || (a.recoveryKinds?.length ?? 0) > 0) ? 1 : 0, + fallover: input.comboTargetAdvanced === true || labels.size > 1 ? 1 : 0, + }; } /** The line, without writing it. */ export function usageLogLine(input: UsageLogInput): string { const { usage } = input; const status = Number.isInteger(input.status) ? String(input.status) : "-"; + const { r429, retry, fallover } = usageLogFlags(input); return `[usage] status=${status} provider=${providerField(input.provider)} model=${idField(input.model)}` + ` in=${count(usage?.inputTokens)} out=${count(usage?.outputTokens)}` + ` cached=${count(usage?.cacheReadInputTokens ?? usage?.cachedInputTokens)}` - + ` key=${hashedKeyId(input.apiKeyId)} ms=${count(input.durationMs)}`; + + ` key=${hashedKeyId(input.apiKeyId)} ms=${count(input.durationMs)}` + + ` ttft=${count(input.firstOutputMs)} acct=${hashedAccount(input)} r429=${r429} retry=${retry} fallover=${fallover}`; } /** Write the one line, on an AgentRadar gateway only. Never throws: logging must not fail a request. */ diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index ab4ec63a400..c1021e7b202 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1641,5 +1641,6 @@ "cli-effort-slug.test.ts": "cli", "grok-47-build-fast-metadata.test.ts": "providers/xai", "upstream-refusal-log.test.ts": "lib", - "call-usage-line-log.test.ts": "lib" + "call-usage-line-log.test.ts": "lib", + "request-summary-line.test.ts": "lib" } diff --git a/tests/lib/call-usage-line-log.test.ts b/tests/lib/call-usage-line-log.test.ts index 778d6eb9786..6e92824336a 100644 --- a/tests/lib/call-usage-line-log.test.ts +++ b/tests/lib/call-usage-line-log.test.ts @@ -14,7 +14,7 @@ const API_KEY_SECRET = "sk-test-9942distinctive"; const AUTH_HEADER = "Bearer access-token-5531distinctive"; const KEY_ID = "key-record-1"; -const LINE_RE = /^\[usage\] status=\d+ provider="[^"]*" model="[^"]*" in=(?:\d+|-) out=(?:\d+|-) cached=(?:\d+|-) key=(?:[0-9a-f]{8}|-) ms=(?:\d+|-)$/; +const LINE_RE = /^\[usage\] status=\d+ provider="[^"]*" model="[^"]*" in=(?:\d+|-) out=(?:\d+|-) cached=(?:\d+|-) key=(?:[0-9a-f]{8}|-) ms=(?:\d+|-) ttft=(?:\d+|-) acct=(?:[0-9a-f]{8}|-) r429=[01] retry=[01] fallover=[01]$/; let infos: string[]; let restoreEnv: () => void; @@ -44,7 +44,7 @@ describe("usage line", () => { }); const expectedHash = createHash("sha256").update(KEY_ID).digest("hex").slice(0, 8); expect(usageLines()).toEqual([ - `[usage] status=200 provider="anthropic" model="claude-test-1" in=1200 out=340 cached=900 key=${expectedHash} ms=2150`, + `[usage] status=200 provider="anthropic" model="claude-test-1" in=1200 out=340 cached=900 key=${expectedHash} ms=2150 ttft=- acct=- r429=0 retry=0 fallover=0`, ]); expect(LINE_RE.test(usageLines()[0]!)).toBe(true); }); @@ -57,7 +57,7 @@ describe("usage line", () => { test("missing counts, key and duration log as a dash", () => { logUsageLine({ status: 502 }); - expect(usageLines()).toEqual(['[usage] status=502 provider="-" model="-" in=- out=- cached=- key=- ms=-']); + expect(usageLines()).toEqual(['[usage] status=502 provider="-" model="-" in=- out=- cached=- key=- ms=- ttft=- acct=- r429=0 retry=0 fallover=0']); expect(hashedKeyId(undefined)).toBe("-"); expect(hashedKeyId("")).toBe("-"); const odd = usageLogLine({ status: 200, usage: { inputTokens: Number.NaN, outputTokens: -3 } }); @@ -148,7 +148,7 @@ describe("usage line from the final request record", () => { try { addFinalRequestLog("ocx-usage-line-2", Date.now(), { model: "gpt-test", provider: "openai", admissionKind: "loopback" }, 499, { closeReason: "client_cancel" }); expect(usageLines()).toHaveLength(1); - expect(usageLines()[0]).toMatch(/^\[usage\] status=499 provider="openai" model="gpt-test" in=- out=- cached=- key=- ms=\d+$/); + expect(usageLines()[0]).toMatch(/^\[usage\] status=499 provider="openai" model="gpt-test" in=- out=- cached=- key=- ms=\d+ ttft=- acct=- r429=0 retry=0 fallover=0$/); } finally { if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; diff --git a/tests/lib/request-summary-line.test.ts b/tests/lib/request-summary-line.test.ts new file mode 100644 index 00000000000..74e1497eb87 --- /dev/null +++ b/tests/lib/request-summary-line.test.ts @@ -0,0 +1,105 @@ +import { createHash } from "node:crypto"; +import { describe, expect, test } from "bun:test"; +import { hashedAccount, usageLogLine, type UsageLogInput } from "../../src/lib/usage-line-log"; + +const sha8 = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 8); +const tag = (label: string) => hashedAccount({ accountLogLabel: label }); + +const PROMPT = "PROMPT-SECRET-please summarise the private contract for jane@example.com"; +const KEY_ID = "key_rec_7f3a91c2d4"; +const ACCT_A = "p2e22d0"; +const ACCT_B = "p9a81c33"; + +function entry(overrides: Partial = {}): UsageLogInput { + return { + status: 200, + provider: "openai", + model: "gpt-5.6-sol", + usage: { inputTokens: 100, outputTokens: 10, cacheReadInputTokens: 40 }, + apiKeyId: KEY_ID, + durationMs: 1234, + firstOutputMs: 321, + accountLogLabel: ACCT_A, + ...overrides, + }; +} + +describe("per-request summary line", () => { + test("keeps the existing fields in order and appends the new ones", () => { + const line = usageLogLine(entry()); + expect(line).toBe( + `[usage] status=200 provider="openai" model="gpt-5.6-sol" in=100 out=10 cached=40` + + ` key=${sha8(KEY_ID)} ms=1234 ttft=321 acct=${tag(ACCT_A)} r429=0 retry=0 fallover=0`, + ); + }); + + test("a 429 then 200 row gives r429=1 retry=1", () => { + const line = usageLogLine(entry({ + spend: { sends: 2 }, + attempts: [ + { status: 429, sendCount: 1, recoveryKinds: ["key-429"], recoveryCount: 1, accountLogLabel: ACCT_A }, + { status: 200, sendCount: 1, recoveryKinds: [], accountLogLabel: ACCT_A }, + ] as UsageLogInput["attempts"], + })); + expect(line).toContain(" r429=1 retry=1 fallover=0"); + }); + + test("a recovery on a single send still counts as a retry", () => { + const line = usageLogLine(entry({ + spend: { sends: 1 }, + attempts: [{ status: 200, recoveryKinds: ["oauth-401"], recoveryCount: 1, accountLogLabel: ACCT_A }], + })); + expect(line).toContain(" r429=0 retry=1 fallover=0"); + }); + + test("two accounts across attempts give fallover=1", () => { + const line = usageLogLine(entry({ + accountLogLabel: ACCT_B, + attempts: [ + { status: 429, accountLogLabel: ACCT_A }, + { status: 200, accountLogLabel: ACCT_B }, + ], + })); + expect(line).toContain(" r429=1 "); + expect(line).toContain(" fallover=1"); + expect(line).toContain(`acct=${tag(ACCT_B)}`); + }); + + test("an advanced combo target gives fallover=1 on one account", () => { + expect(usageLogLine(entry({ comboTargetAdvanced: true }))).toContain(" fallover=1"); + }); + + test("the account tag is keyed: stable for one label, distinct across labels, never the plain sha256", () => { + expect(tag(ACCT_A)).toMatch(/^[0-9a-f]{8}$/); + expect(tag(ACCT_A)).toBe(tag(ACCT_A)); + expect(tag(ACCT_A)).not.toBe(tag(ACCT_B)); + // A plain short hash of a guessable label could be reversed by trying every label. + expect(tag(ACCT_A)).not.toBe(sha8(ACCT_A)); + expect(usageLogLine(entry())).not.toContain(sha8(ACCT_A)); + }); + + test("a missing firstOutputMs gives ttft=- and a missing account gives acct=-", () => { + const line = usageLogLine(entry({ firstOutputMs: undefined, accountLogLabel: undefined })); + expect(line).toContain(" ttft=- acct=- "); + }); + + test("the line never carries the raw key id, the raw account label or prompt text", () => { + const line = usageLogLine({ + ...entry({ + attempts: [ + { status: 429, accountLogLabel: ACCT_A }, + { status: 200, accountLogLabel: ACCT_B }, + ], + }), + // Fields the line must not read, even when a caller's wider record carries them. + prompt: PROMPT, + upstreamError: PROMPT, + email: "jane@example.com", + } as UsageLogInput); + for (const secret of [KEY_ID, ACCT_A, ACCT_B, PROMPT, "PROMPT-SECRET", "jane@example.com"]) { + expect(line).not.toContain(secret); + } + expect(line).toContain(`key=${sha8(KEY_ID)}`); + expect(line).toContain(`acct=${tag(ACCT_B)}`); + }); +});