Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion scripts/test-layout/layout.json
Original file line number Diff line number Diff line change
Expand Up @@ -1810,7 +1810,8 @@
"zz-ci-storage-policy-isolation.test.ts": "ci-workflows",
"zz-pr-coderabbit-readiness-revalidation.test.ts": "ci-workflows",
"upstream-refusal-log.test.ts": "lib",
"call-usage-line-log.test.ts": "lib"
"call-usage-line-log.test.ts": "lib",
"request-summary-line.test.ts": "lib"
},
"migrated": [
"adapters",
Expand Down
64 changes: 57 additions & 7 deletions src/lib/usage-line-log.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { createHash } from "node:crypto";
import { createHash, createHmac, randomBytes } from "node:crypto";
import { baseProviderLabel } from "../providers/label";
import { sanitizeLogMetadataString } from "./redact";

Expand All @@ -7,9 +7,11 @@ import { sanitizeLogMetadataString } from "./redact";
* the share of calls and tokens on each model from the process log alone.
*
* The line carries only the HTTP status, the provider and model ids, three token counts, an 8-hex
* hash of the gateway API key's record id and the duration. It never carries message text, a
* request or response body, a header, a secret key value, an account label or an email: the input is
* the fork's own final request record, and only the fields named below are read from it.
* hash of the gateway API key's record id, the duration, the time to first token, an 8-hex keyed tag of
* the serving account's label and three 0/1 flags (a 429 was seen, a retry happened, the request
* moved to another account or combo target). It never carries message text, a request or response
* body, a header, a secret key value, an account label in clear or an email: the input is the
* fork's own final request record, and only the fields named below are read from it.
*
* It is written only on an AgentRadar gateway (`OPENCODEX_AGENTRADAR=1`, the condition the
* `[ar-acct]` line uses), so a self-hosted gateway gains no new output.
Expand All @@ -32,6 +34,21 @@ export interface UsageLogInput {
/** The configured key's record id (never the secret key value). Absent for environment or loopback admission. */
apiKeyId?: string;
durationMs?: number;
/** Ms from request start to the first non-empty output delta; absent for non-streaming or tool-only calls. */
firstOutputMs?: number;
/** The serving account's stable non-PII label; only its 8-hex hash is logged. */
accountLogLabel?: string;
/** True only when combo execution advanced to another configured target. */
comboTargetAdvanced?: boolean;
/** Upstream sends for the whole logical request; more than one means a retry. */
spend?: { sends?: number };
/** One row per physical attempt; only status, recovery counts and the account label are read. */
attempts?: readonly {
status?: number;
recoveryKinds?: readonly unknown[];
recoveryCount?: number;
accountLogLabel?: string;
}[];
}

/** The OAuth (`o`) and API-key (`k`) account labels `baseProviderLabel` leaves in place. */
Expand All @@ -54,20 +71,53 @@ function count(value: number | undefined): string {
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? String(Math.round(value)) : "-";
}

/** First 8 hex of sha256 of a non-empty id, or "-" when there is none. */
function hash8(id: string | undefined): string {
if (typeof id !== "string" || id === "") return "-";
return createHash("sha256").update(id).digest("hex").slice(0, 8);
}

/** First 8 hex of sha256 of the key record id, or "-" when the call carried no configured key. */
export function hashedKeyId(apiKeyId: string | undefined): string {
if (typeof apiKeyId !== "string" || apiKeyId === "") return "-";
return createHash("sha256").update(apiKeyId).digest("hex").slice(0, 8);
return hash8(apiKeyId);
}

/**
* Account labels are guessable (a provider name plus a short hex suffix), so a plain short hash could be
* reversed by trying them all. The tag is an HMAC under a random key made once per process and never
* written anywhere: the same account gets the same tag for the life of the process, and nothing more.
*/
const ACCOUNT_TAG_KEY = randomBytes(32);

/** First 8 hex of the keyed tag of the serving account label (the entry's, else the last attempt's), or "-". */
export function hashedAccount(input: Pick<UsageLogInput, "accountLogLabel" | "attempts">): string {
const last = [...(input.attempts ?? [])].reverse().find(a => typeof a.accountLogLabel === "string" && a.accountLogLabel !== "");
const label = input.accountLogLabel || last?.accountLogLabel;
if (typeof label !== "string" || label === "") return "-";
return createHmac("sha256", ACCOUNT_TAG_KEY).update(label).digest("hex").slice(0, 8);
}

/** 429 / retry / fallover as 0 or 1, read from the attempt rows, spend total and combo flag only. */
export function usageLogFlags(input: UsageLogInput): { r429: 0 | 1; retry: 0 | 1; fallover: 0 | 1 } {
const attempts = input.attempts ?? [];
const labels = new Set(attempts.map(a => a.accountLogLabel).filter((l): l is string => typeof l === "string" && l !== ""));
return {
r429: input.status === 429 || attempts.some(a => a.status === 429) ? 1 : 0,
retry: (input.spend?.sends ?? 0) > 1 || attempts.some(a => (a.recoveryCount ?? 0) > 0 || (a.recoveryKinds?.length ?? 0) > 0) ? 1 : 0,
fallover: input.comboTargetAdvanced === true || labels.size > 1 ? 1 : 0,
};
}

/** The line, without writing it. */
export function usageLogLine(input: UsageLogInput): string {
const { usage } = input;
const status = Number.isInteger(input.status) ? String(input.status) : "-";
const { r429, retry, fallover } = usageLogFlags(input);
return `[usage] status=${status} provider=${providerField(input.provider)} model=${idField(input.model)}`
+ ` in=${count(usage?.inputTokens)} out=${count(usage?.outputTokens)}`
+ ` cached=${count(usage?.cacheReadInputTokens ?? usage?.cachedInputTokens)}`
+ ` key=${hashedKeyId(input.apiKeyId)} ms=${count(input.durationMs)}`;
+ ` key=${hashedKeyId(input.apiKeyId)} ms=${count(input.durationMs)}`
+ ` ttft=${count(input.firstOutputMs)} acct=${hashedAccount(input)} r429=${r429} retry=${retry} fallover=${fallover}`;
}

/** Write the one line, on an AgentRadar gateway only. Never throws: logging must not fail a request. */
Expand Down
3 changes: 2 additions & 1 deletion tests/fixtures/test-layout-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -1641,5 +1641,6 @@
"cli-effort-slug.test.ts": "cli",
"grok-47-build-fast-metadata.test.ts": "providers/xai",
"upstream-refusal-log.test.ts": "lib",
"call-usage-line-log.test.ts": "lib"
"call-usage-line-log.test.ts": "lib",
"request-summary-line.test.ts": "lib"
}
8 changes: 4 additions & 4 deletions tests/lib/call-usage-line-log.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const API_KEY_SECRET = "sk-test-9942distinctive";
const AUTH_HEADER = "Bearer access-token-5531distinctive";
const KEY_ID = "key-record-1";

const LINE_RE = /^\[usage\] status=\d+ provider="[^"]*" model="[^"]*" in=(?:\d+|-) out=(?:\d+|-) cached=(?:\d+|-) key=(?:[0-9a-f]{8}|-) ms=(?:\d+|-)$/;
const LINE_RE = /^\[usage\] status=\d+ provider="[^"]*" model="[^"]*" in=(?:\d+|-) out=(?:\d+|-) cached=(?:\d+|-) key=(?:[0-9a-f]{8}|-) ms=(?:\d+|-) ttft=(?:\d+|-) acct=(?:[0-9a-f]{8}|-) r429=[01] retry=[01] fallover=[01]$/;

let infos: string[];
let restoreEnv: () => void;
Expand Down Expand Up @@ -44,7 +44,7 @@ describe("usage line", () => {
});
const expectedHash = createHash("sha256").update(KEY_ID).digest("hex").slice(0, 8);
expect(usageLines()).toEqual([
`[usage] status=200 provider="anthropic" model="claude-test-1" in=1200 out=340 cached=900 key=${expectedHash} ms=2150`,
`[usage] status=200 provider="anthropic" model="claude-test-1" in=1200 out=340 cached=900 key=${expectedHash} ms=2150 ttft=- acct=- r429=0 retry=0 fallover=0`,
]);
expect(LINE_RE.test(usageLines()[0]!)).toBe(true);
});
Expand All @@ -57,7 +57,7 @@ describe("usage line", () => {

test("missing counts, key and duration log as a dash", () => {
logUsageLine({ status: 502 });
expect(usageLines()).toEqual(['[usage] status=502 provider="-" model="-" in=- out=- cached=- key=- ms=-']);
expect(usageLines()).toEqual(['[usage] status=502 provider="-" model="-" in=- out=- cached=- key=- ms=- ttft=- acct=- r429=0 retry=0 fallover=0']);
expect(hashedKeyId(undefined)).toBe("-");
expect(hashedKeyId("")).toBe("-");
const odd = usageLogLine({ status: 200, usage: { inputTokens: Number.NaN, outputTokens: -3 } });
Expand Down Expand Up @@ -148,7 +148,7 @@ describe("usage line from the final request record", () => {
try {
addFinalRequestLog("ocx-usage-line-2", Date.now(), { model: "gpt-test", provider: "openai", admissionKind: "loopback" }, 499, { closeReason: "client_cancel" });
expect(usageLines()).toHaveLength(1);
expect(usageLines()[0]).toMatch(/^\[usage\] status=499 provider="openai" model="gpt-test" in=- out=- cached=- key=- ms=\d+$/);
expect(usageLines()[0]).toMatch(/^\[usage\] status=499 provider="openai" model="gpt-test" in=- out=- cached=- key=- ms=\d+ ttft=- acct=- r429=0 retry=0 fallover=0$/);
} finally {
if (previousHome === undefined) delete process.env.OPENCODEX_HOME;
else process.env.OPENCODEX_HOME = previousHome;
Expand Down
105 changes: 105 additions & 0 deletions tests/lib/request-summary-line.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import { createHash } from "node:crypto";
import { describe, expect, test } from "bun:test";
import { hashedAccount, usageLogLine, type UsageLogInput } from "../../src/lib/usage-line-log";

const sha8 = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 8);
const tag = (label: string) => hashedAccount({ accountLogLabel: label });

const PROMPT = "PROMPT-SECRET-please summarise the private contract for jane@example.com";
const KEY_ID = "key_rec_7f3a91c2d4";
const ACCT_A = "p2e22d0";
const ACCT_B = "p9a81c33";

function entry(overrides: Partial<UsageLogInput> = {}): UsageLogInput {
return {
status: 200,
provider: "openai",
model: "gpt-5.6-sol",
usage: { inputTokens: 100, outputTokens: 10, cacheReadInputTokens: 40 },
apiKeyId: KEY_ID,
durationMs: 1234,
firstOutputMs: 321,
accountLogLabel: ACCT_A,
...overrides,
};
}

describe("per-request summary line", () => {
test("keeps the existing fields in order and appends the new ones", () => {
const line = usageLogLine(entry());
expect(line).toBe(
`[usage] status=200 provider="openai" model="gpt-5.6-sol" in=100 out=10 cached=40`
+ ` key=${sha8(KEY_ID)} ms=1234 ttft=321 acct=${tag(ACCT_A)} r429=0 retry=0 fallover=0`,
);
});

test("a 429 then 200 row gives r429=1 retry=1", () => {
const line = usageLogLine(entry({
spend: { sends: 2 },
attempts: [
{ status: 429, sendCount: 1, recoveryKinds: ["key-429"], recoveryCount: 1, accountLogLabel: ACCT_A },
{ status: 200, sendCount: 1, recoveryKinds: [], accountLogLabel: ACCT_A },
] as UsageLogInput["attempts"],
}));
expect(line).toContain(" r429=1 retry=1 fallover=0");
});

test("a recovery on a single send still counts as a retry", () => {
const line = usageLogLine(entry({
spend: { sends: 1 },
attempts: [{ status: 200, recoveryKinds: ["oauth-401"], recoveryCount: 1, accountLogLabel: ACCT_A }],
}));
expect(line).toContain(" r429=0 retry=1 fallover=0");
});

test("two accounts across attempts give fallover=1", () => {
const line = usageLogLine(entry({
accountLogLabel: ACCT_B,
attempts: [
{ status: 429, accountLogLabel: ACCT_A },
{ status: 200, accountLogLabel: ACCT_B },
],
}));
expect(line).toContain(" r429=1 ");
expect(line).toContain(" fallover=1");
expect(line).toContain(`acct=${tag(ACCT_B)}`);
});

test("an advanced combo target gives fallover=1 on one account", () => {
expect(usageLogLine(entry({ comboTargetAdvanced: true }))).toContain(" fallover=1");
});

test("the account tag is keyed: stable for one label, distinct across labels, never the plain sha256", () => {
expect(tag(ACCT_A)).toMatch(/^[0-9a-f]{8}$/);
expect(tag(ACCT_A)).toBe(tag(ACCT_A));
expect(tag(ACCT_A)).not.toBe(tag(ACCT_B));
// A plain short hash of a guessable label could be reversed by trying every label.
expect(tag(ACCT_A)).not.toBe(sha8(ACCT_A));
expect(usageLogLine(entry())).not.toContain(sha8(ACCT_A));
});

test("a missing firstOutputMs gives ttft=- and a missing account gives acct=-", () => {
const line = usageLogLine(entry({ firstOutputMs: undefined, accountLogLabel: undefined }));
expect(line).toContain(" ttft=- acct=- ");
});

test("the line never carries the raw key id, the raw account label or prompt text", () => {
const line = usageLogLine({
...entry({
attempts: [
{ status: 429, accountLogLabel: ACCT_A },
{ status: 200, accountLogLabel: ACCT_B },
],
}),
// Fields the line must not read, even when a caller's wider record carries them.
prompt: PROMPT,
upstreamError: PROMPT,
email: "jane@example.com",
} as UsageLogInput);
for (const secret of [KEY_ID, ACCT_A, ACCT_B, PROMPT, "PROMPT-SECRET", "jane@example.com"]) {
expect(line).not.toContain(secret);
}
expect(line).toContain(`key=${sha8(KEY_ID)}`);
expect(line).toContain(`acct=${tag(ACCT_B)}`);
});
});