From 0da4673a6393deda0e531a0177e4db1467b7520f Mon Sep 17 00:00:00 2001 From: Vadym O Date: Wed, 18 Nov 2026 15:09:09 +0800 Subject: [PATCH] fix(gateway): enforce Anthropic physical-send ceilings and gate quota regressions Reserve Anthropic OAuth inference at the existing physical dispatch boundary. Share inherited request ledgers and pending-hop permits instead of bypassing ceilings or double charging helper reports. Preserve the fork default pool walk without widening custom policies. Add hard zero/one-send, funded replacement, inherited budget, continuation and image-sidecar regression cases, stable cache-prefix control and pinned quota-400 isolation. Correct direct-handler terminal fixtures to own the spend ledger. Run the offline gateway suite inside the existing required CI gates job; fork publication retains its exact-SHA push-CI requirement. Co-Authored-By: Claude Code --- .github/workflows/ci.yml | 3 + CONTRIBUTING.md | 12 +++ package.json | 1 + src/server/responses/adapter-continuation.ts | 7 +- src/server/responses/adapter-dispatch.ts | 24 +++-- src/server/responses/request-transport.ts | 33 +++++- src/server/responses/sidecar-execution.ts | 13 ++- structure/transports/responses-failover.md | 4 +- structure/transports/responses-spend.md | 14 +++ structure/transports/responses.md | 3 +- .../anthropic-quota-dispatch.test.ts | 100 ++++++++++++++++++ .../anthropic-replay-oauth-identity.test.ts | 14 +++ .../server/agentradar-account-pin-e2e.test.ts | 19 +++- tests/server/terminal-guard-server.test.ts | 6 ++ 14 files changed, 228 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72af24c514c..04d64d0ef7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -627,6 +627,9 @@ jobs: cd gui bun install --frozen-lockfile + - name: Gateway quota and cache guardrails + run: bun run test:gateway-quota + - name: GUI lint if: needs.changes.outputs.gui == 'true' run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dce3eb38ae7..efbc6f21dd4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,18 @@ Source development requires the `bun` CLI on your `PATH`. The published npm pack Bun runtime for end users, but contributor commands such as `bun install`, `bun run test`, and `bun run prepush` run from your local Bun installation. +## AgentRadar fork quota and cache checks + +On the hosted fork release line, run `bun run test:gateway-quota` before proposing gateway changes. +It runs the existing offline account-pin, quota-refusal, physical-send, replay-prefix, usage and +cache-stabilization suites together through the isolated test runner. Provider traffic is synthetic; +these checks do not measure a live cache-hit rate or spend a subscription. + +The `Gateway quota and cache guardrails` step is part of the existing Cross-platform CI `gates` +job, so a failing suite fails the aggregate `ci` check. Fork publication already requires successful +push-event Cross-platform CI for the exact release SHA. This does not replace the broader tests, +review or the repository's branch rules. + ## Pull request contract A ready-for-review PR is the author's claim that the change is complete, understood, tested, and suitable for merging. Opening a PR does not transfer responsibility for the branch to maintainers. diff --git a/package.json b/package.json index b4c177a8cc9..17363fec6ef 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "start": "bun run src/cli/index.ts start", "test": "bun scripts/test.ts", "test:changed": "bun scripts/test.ts --changed=dev", + "test:gateway-quota": "bun scripts/test.ts tests/adapters/anthropic/anthropic-quota-dispatch.test.ts tests/adapters/anthropic/anthropic-pool-extra-usage-400.test.ts tests/adapters/anthropic/anthropic-account-pool.test.ts tests/adapters/anthropic/anthropic-replay-oauth-identity.test.ts tests/adapters/anthropic/anthropic-replay-combo.test.ts tests/server/agentradar-account-pin-e2e.test.ts tests/responses/responses-send-budget-counts.test.ts tests/adapters/adapter-inner-send-budget.test.ts tests/usage/key-attribution.test.ts tests/claude-integration/claude-inbound-cache-stabilize.test.ts tests/server/terminal-guard-server.test.ts", "typecheck": "bun x tsc --noEmit", "audit:high": "bun audit --audit-level=high && cd gui && bun audit --audit-level=high", "privacy:scan": "bun scripts/privacy-scan.ts", diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 017ab1dcfec..3847969f3cd 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -90,6 +90,8 @@ export function createAdapterContinuations( | "applyFailoverSnapshot" | "replayOAuthCredentialSnapshot" | "noteRoutedAttemptSend" + | "anthropicPhysicalBudget" + | "anthropicSendAvailable" >, sidecarState: Pick, sendBudgetState: Pick< @@ -257,7 +259,7 @@ export function createAdapterContinuations( ...(continuationTransientPolicy ? { attempts: remainingTransientSendBudget(continuationTransientPolicy.attempts), - onSendsConsumed: noteTransientSends, + onSendsConsumed: transportState.anthropicPhysicalBudget ? undefined : noteTransientSends, } : {}), }, @@ -283,7 +285,7 @@ export function createAdapterContinuations( // A credential denial (403 oauth_org_not_allowed) rotates the Anthropic account here too: // the continuation is a fresh send, so a denied credential is recoverable rather than // terminal. `nextParsed` is the terminal-guard clone; both owners are rebound. - const deniedCredentialRecovery = await rotateAnthropicProviderOnCredentialDenial({ + const deniedCredentialRecovery = !transportState.anthropicSendAvailable() ? null : await rotateAnthropicProviderOnCredentialDenial({ config, route, inboundWire, logCtx, transportState, parsed, retryParsed: nextParsed, response, @@ -391,6 +393,7 @@ export function createAdapterContinuations( transportState.anthropicPoolAccountId && !isNonReplayableResponse(response) && transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST + && transportState.anthropicSendAvailable() ) { try { const peeked = await peekAnthropicPoolRotation(response, upstream.signal); diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 560bdd8c6f8..d58fe4441cf 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -129,6 +129,9 @@ export async function prepareAdapterExchange( | "genericFailovers" | "applyFailoverSnapshot" | "noteRoutedAttemptSend" + | "anthropicPhysicalBudget" + | "anthropicSendAvailable" + | "setAnthropicDispatchBudget" >, responseEffects: Pick, sendBudgetState: Pick< @@ -178,7 +181,7 @@ export async function prepareAdapterExchange( sendBudgetExhausted, reserveCredentialHop, } = sendBudgetState; - + transportState.setAnthropicDispatchBudget(adapterDispatchBudget); const upstream = new AbortController(); const cleanupUpstreamAbort = linkAbortSignal(upstream, options.abortSignal); @@ -359,7 +362,7 @@ export async function prepareAdapterExchange( // shared counter was inherited without ever being read as a limit. ? { attempts: remainingTransientSendBudget(transientPolicy.attempts), - onSendsConsumed: noteTransientSends, + onSendsConsumed: transportState.anthropicPhysicalBudget ? undefined : noteTransientSends, } : {}), }, @@ -455,7 +458,7 @@ export async function prepareAdapterExchange( // their physical-send boundary, without booking the request ledger again. let fastDowngradeWorkflowCharged = false; const chargeFastDowngradeWorkflowSend = (): void => { - if (recovery !== "anthropic-fast-downgrade" || fastDowngradeWorkflowCharged) return; + if (transportState.anthropicPhysicalBudget || recovery !== "anthropic-fast-downgrade" || fastDowngradeWorkflowCharged) return; fastDowngradeWorkflowCharged = true; chargeWorkflowSends(sendBudgetState.workflowRootId, 1); }; @@ -537,7 +540,7 @@ export async function prepareAdapterExchange( // Same rule as the passthrough rebuild: spend the base allowance first, then the one // shared final-recovery reserve, so a recovery that follows a spent streak still gets // its single send instead of dying at three. - const refetchAllowance = refetchTransientPolicy + const refetchAllowance = refetchTransientPolicy && !transportState.anthropicPhysicalBudget ? recoverySendAllowance( refetchTransientPolicy.attempts, recoveryClassFor(recovery), @@ -571,7 +574,7 @@ export async function prepareAdapterExchange( ...(refetchAllowance ? { attempts: refetchAllowance.attempts, - onSendsConsumed: noteTransientSends, + onSendsConsumed: transportState.anthropicPhysicalBudget ? undefined : noteTransientSends, } : {}), }, @@ -704,7 +707,7 @@ export async function prepareAdapterExchange( // An Anthropic OAuth credential denial (403 oauth_org_not_allowed) is not a quota verdict: // the account is marked for reauth and the turn continues on the next eligible account. - const deniedCredentialRecovery = await rotateAnthropicProviderOnCredentialDenial({ + const deniedCredentialRecovery = !transportState.anthropicSendAvailable() ? null : await rotateAnthropicProviderOnCredentialDenial({ config, route, inboundWire, logCtx, transportState, parsed, response: upstreamResponse, signal: upstream.signal, @@ -735,8 +738,8 @@ export async function prepareAdapterExchange( const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined; const hop = reserveCredentialHop( "repair", - `${route.providerName}|${route.modelId}|anthropic-fast-downgrade`, - !adapterOwnsDispatch && transientRetryPolicyFor(route.provider) !== null, + `${route.providerName}|${route.modelId}|${transportState.anthropicPhysicalBudget ? "anthropic-physical" : "anthropic-fast-downgrade"}`, + !transportState.anthropicPhysicalBudget && !adapterOwnsDispatch && transientRetryPolicyFor(route.provider) !== null, ); if (hop.allowed) { anthropicFastDowngradeGuard.attempted = true; @@ -751,7 +754,7 @@ export async function prepareAdapterExchange( let result: Response | { failed: Response }; try { result = await rebuildAndRefetch("anthropic-fast-downgrade", () => { - if (!adapterOwnsDispatch) hop.permit?.use(); + if (!adapterOwnsDispatch && !transportState.anthropicPhysicalBudget) hop.permit?.use(); }); } finally { sendBudgetState.pendingHopPermit = undefined; @@ -869,6 +872,7 @@ export async function prepareAdapterExchange( while ( transportState.anthropicPoolAccountId && transportState.anthropicPoolFailovers < ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST + && transportState.anthropicSendAvailable() ) { let poolRotation: Awaited>; try { @@ -1012,7 +1016,7 @@ export async function prepareAdapterExchange( // and settling here first would hand it a dead permit, which it reads as an // exhausted request and stops sending on. result = await rebuildAndRefetch("oauth-account-429", () => { - if (!adapterOwnsDispatch) hop.permit?.use(); + if (!adapterOwnsDispatch && !transportState.anthropicPhysicalBudget) hop.permit?.use(); }); } finally { sendBudgetState.pendingHopPermit = undefined; diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index 0a8948ba725..91c3eb7a022 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -1,4 +1,7 @@ import type { ResponsesRequestContext, ResponsesAdmissionState } from "./core-options"; +import { CODEX_TEXT_GUARDED_BUDGET_POLICY, deriveRequestExecutionBudget, isRequestExecutionBudget, type RequestExecutionBudget } from "../../lib/request-execution-budget"; +import { SendBudgetExhaustedError } from "../../lib/upstream-retry"; +import { chargeWorkflowSends } from "../../lib/workflow-budget"; import type { PreparedResponsesRequest } from "./request-prepare"; import type { OAuthAccessSnapshot } from "../../oauth"; import { @@ -11,6 +14,7 @@ import type { ProviderAdapter, AdapterRequest } from "../../adapters/base"; import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig, OcxUsage } from "../../types"; import type { AnthropicAccountSelectionReason } from "../../oauth/anthropic-routing"; import { + ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, isAnthropicAccountPoolEnabled, getAnthropicPoolAccessSnapshot, commitAnthropicSelectionRouting, @@ -90,6 +94,18 @@ export async function prepareResponsesTransport( ) { const { config, logCtx, options, req } = requestContext; const { route, parsed, inboundWire, translatorBudget } = requestState; + const anthropicPhysicalBudget = route.providerName === "anthropic" && route.provider.authMode === "oauth"; + if (anthropicPhysicalBudget && isRequestExecutionBudget(options.sendBudget) + && options.sendBudget.policy === CODEX_TEXT_GUARDED_BUDGET_POLICY) { + // Keep the fork's existing 16-account recovery allowance; never widen an explicit policy. + options.sendBudget = deriveRequestExecutionBudget(options.sendBudget, { + ...options.sendBudget.policy, maxTotalModelSends: ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST + 1, + baseSendAllowance: ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST + 1, + }); + } + let anthropicDispatchBudget = isRequestExecutionBudget(options.sendBudget) ? options.sendBudget : undefined; + const anthropicSendAvailable = (): boolean => !anthropicPhysicalBudget || !anthropicDispatchBudget + || anthropicDispatchBudget.used < anthropicDispatchBudget.policy.maxTotalModelSends; // AgentRadar image patch: the pinned account's own snapshot and credential generation are the // only admission; the global selection, affinity and pool cursors are never read or written. const pin = route.provider.authMode === "oauth" ? options.agentRadarAccount : undefined; @@ -275,11 +291,11 @@ export async function prepareResponsesTransport( // physical dispatch binds it to the selection that actually reaches the upstream. let pendingKeySend: { estimate: number | undefined; recovery?: AttemptRecoveryKind } | undefined; const noteRoutedAttemptSend = (estimate: number | undefined, recovery?: AttemptRecoveryKind): void => { - if (usesApiKeyAccount(route.provider)) pendingKeySend = { estimate, recovery }; + if (anthropicPhysicalBudget || usesApiKeyAccount(route.provider)) pendingKeySend = { estimate, recovery }; else noteProviderAttemptSend(logCtx, route.providerName, route.provider, estimate, recovery); }; const commitKeyAttemptSend = (): void => { - if (!usesApiKeyAccount(route.provider)) return; + if (!anthropicPhysicalBudget && !usesApiKeyAccount(route.provider)) return; noteProviderAttemptSend(logCtx, route.providerName, route.provider, pendingKeySend?.estimate ?? logCtx.usageLogInputTokens, pendingKeySend?.recovery); pendingKeySend = undefined; @@ -453,11 +469,19 @@ export async function prepareResponsesTransport( // Reselection can choose a provider override instead of the supplied executor. // Either way the send crosses the physical boundary, so the connection policy is // applied around whichever implementation was just selected (#4992). - commitKeyAttemptSend(); // The binding travels with the send, so a rebuilt request resolves its provider route // against the destination it is actually going to rather than the one this dispatch // started with. Account reselection can move the upstream host, which would otherwise // apply a host-scoped decision to a different host. + if (anthropicPhysicalBudget && anthropicDispatchBudget) { + const dispatch = anthropicDispatchBudget.reserveDispatch({ + sendClass: anthropicDispatchBudget.used === 0 ? "initial" : "auth-recovery", + targetKey: `${route.providerName}|${route.modelId}|anthropic-physical`, + }); + if (!dispatch.allowed || !dispatch.permit.use()) throw new SendBudgetExhaustedError("Anthropic"); + chargeWorkflowSends(req.headers.get("x-codex-parent-thread-id")?.trim() || undefined, 1); + } + commitKeyAttemptSend(); const response = await sendWithConnectionPolicy( fetchImpl, destination, @@ -912,6 +936,9 @@ export async function prepareResponsesTransport( anthropicMessagesSource = undefined; }, oauthDispatch, + anthropicPhysicalBudget, + anthropicSendAvailable, + setAnthropicDispatchBudget(budget: RequestExecutionBudget | undefined): void { anthropicDispatchBudget = budget; }, noteRoutedAttemptSend, commitKeyAttemptSend, bindKeyUsageFromBridge, diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 9d19e32428f..8855e9b6687 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -72,6 +72,9 @@ export async function executeResponsesSidecars( | "noteRoutedAttemptSend" | "bindKeyUsageFromBridge" | "exactAnthropicSourceReplay" + | "anthropicPhysicalBudget" + | "anthropicSendAvailable" + | "setAnthropicDispatchBudget" >, sidecarState: Pick, responseEffects: Pick< @@ -81,7 +84,7 @@ export async function executeResponsesSidecars( | "notifyResponseComplete" | "cancelResponseCompletion" >, - sendBudgetState: Pick, + sendBudgetState: Pick, ) { const { config, options, logCtx } = requestContext; const { @@ -101,6 +104,7 @@ export async function executeResponsesSidecars( } = requestState; const { routedCompaction, openAiSidecar } = sidecarState; const { reserveCredentialHop } = sendBudgetState; + transportState.setAnthropicDispatchBudget(sendBudgetState.adapterDispatchBudget); const { commitReasoningReplayServingRoute, continuationStateForResponse, @@ -237,10 +241,9 @@ export async function executeResponsesSidecars( ) { // Same intersection for the Anthropic roster: its own per-request bound still applies, // and the shared budget decides whether this request may spend another send at all. - const hop = reserveCredentialHop( - "auth-recovery", - `${route.providerName}|${route.modelId}|sidecar-anthropic-429`, - ); + const hop = transportState.anthropicPhysicalBudget + ? { allowed: transportState.anthropicSendAvailable(), permit: undefined } + : reserveCredentialHop("auth-recovery", `${route.providerName}|${route.modelId}|sidecar-anthropic-429`); if (!hop.allowed) return null; const nextAccountId = rotateAnthropicAccountOn429( config, diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index 4d26d74453f..61fd8762b52 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -338,7 +338,9 @@ In `adapter-dispatch.ts` the guard at the top of the recovery loop is necessary sufficient. The refusal can also be produced by a refetch made INSIDE an arm, and that arm then still holds it: the same-target loop re-enters while `rateLimitRetries` is below the configured attempts, and the key, Anthropic-pool and generic-OAuth rotations re-enter while a -credential is left to try. The key-401 arm is in the same class from the other direction — its +credential is left to try. Hosted Anthropic rotations also follow the +[shared physical-send contract](responses-spend.md#hosted-anthropic-physical-sends): no funded +replacement means the current refusal remains authoritative. The key-401 arm is in the same class from the other direction — its refetch answers 429 and it falls through into the arms below. So every arm that reassigns `upstreamResponse` from `rebuildAndRefetch` re-enters the loop guard rather than continuing, which is what makes the top-of-loop check the single exit for this verdict. diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index c749375a83b..1b69167ee0b 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -74,6 +74,20 @@ derivation agree rather than each settling against a counter the other cannot se What stays per-scope is deliberate: the reserve, alternate-target and transition ledgers are each target's own recovery decision, while the physical-send total is what binds every target together. +## Hosted Anthropic physical sends + +The fork charges Anthropic OAuth inference at the `oauthDispatch` physical boundary in +`src/server/responses/request-transport.ts`, including initial, continuation and sidecar sends. +The boundary uses the shared adapter budget view, so an existing hop reservation is consumed +rather than charged twice; helper reporters do not charge that same send again. A zero allowance +prevents inference, and an unfunded account replacement retains the preceding upstream refusal. +Only the factory's unchanged default guarded policy is expanded for the fork's existing account +walk. Explicit policies and inherited custom ceilings are never widened or reset. +`tests/adapters/anthropic/anthropic-quota-dispatch.test.ts` counts mock-provider calls for zero, +one-send, funded replacement, inherited-budget, continuation and sidecar cases. The fork's +`bun run test:gateway-quota` also exercises pin isolation, replay cache-prefix stability and usage +accounting in the existing CI `gates` job; it makes no live cache-hit or paid-provider claim. + ## Durable spend reservations The request's send budget bounds how many times it may reach upstream; the spend ledger bounds diff --git a/structure/transports/responses.md b/structure/transports/responses.md index d5aeacf7151..0361b0d0e7d 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -442,7 +442,8 @@ remain live. Owner names are distinct from local decision variables: `admissionS lease while a block-local `admission` holds only the acquisition result. `handleResponses` creates or inherits the same logical-request send holder. The budget owner -reads that holder rather than minting a per-phase allowance. Combo recursion is injected through +reads that holder rather than minting a per-phase allowance. Hosted Anthropic OAuth sends follow +[the physical dispatch accounting contract](responses-spend.md#hosted-anthropic-physical-sends). Combo recursion is injected through `ResponsesDispatchers`: a child re-enters the public handler without a reverse runtime import from the combo implementation into `core.ts`. `core-lifetime.ts` owns the shared run-turn response marker and translator-budget finalization, so the combo and delivery paths observe one identity. diff --git a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts index 3e641d93713..4886710a539 100644 --- a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts +++ b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts @@ -12,6 +12,7 @@ import { clearResponseStateForTests } from "../../../src/responses/state"; import { handleResponses } from "../../../src/server/responses"; import type { OcxConfig, OcxProviderConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { createRequestExecutionBudget } from "../../../src/lib/request-execution-budget"; const actualResolver = await import("../../../src/server/adapter-resolve"); const actualResolveAdapter = actualResolver.resolveAdapter; @@ -163,6 +164,105 @@ function post(config: OcxConfig, body: Record = {}) { }), config, { model: "", provider: "" }); } +test.each([400, 429])("gateway quota guard: HTTP %i account rotation cannot exceed an explicit physical-send ceiling", async status => { + await seed(3); + const config = configFor(() => status === 429 ? limited() : Response.json({ type: "error", + error: { type: "invalid_request_error", message: "You're out of extra usage." } }, { status })); + config.anthropicAccountPool = { enabled: true, strategy: "round-robin" }; + const budget = createRequestExecutionBudget({ maxTotalModelSends: 1, baseSendAllowance: 1, + finalRecoveryAllowance: 0, maxAlternateTargetSends: 0, maxTargetTransitions: 0 }); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", input: "Answer briefly", stream: false }), + }), config, { model: "", provider: "" }, { sendBudget: budget }); + const text = await response.text(); + expect(text).toContain(status === 400 ? "out of extra usage" : "quota exhausted"); + expect(text).not.toContain("request_send_budget_exhausted"); + expect(sent).toHaveLength(1); + expect(budget.used).toBe(1); +}); + +test("gateway quota guard: a funded replacement sends and charges exactly once", async () => { + await seed(2); + const config = configFor(body => sent.length === 1 ? limited() : answer(body.stream === true)); + const budget = createRequestExecutionBudget({ maxTotalModelSends: 2, baseSendAllowance: 2, + finalRecoveryAllowance: 0, maxAlternateTargetSends: 0, maxTargetTransitions: 0 }); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", input: "Answer briefly", stream: false }), + }), config, { model: "", provider: "" }, { sendBudget: budget }); + expect(await response.text()).toContain("The answer is complete."); + expect(sent).toHaveLength(2); + expect(budget.used).toBe(2); +}); + +test("gateway quota guard: inherited usage is not reset by Anthropic dispatch", async () => { + await seed(2); + const config = configFor(() => limited()); + const budget = createRequestExecutionBudget({ maxTotalModelSends: 2, baseSendAllowance: 2, + finalRecoveryAllowance: 0, maxAlternateTargetSends: 0, maxTargetTransitions: 0 }); + budget.used = 1; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", input: "Answer briefly", stream: false }), + }), config, { model: "", provider: "" }, { sendBudget: budget }); + expect(await response.text()).toContain("quota"); + expect(sent).toHaveLength(1); + expect(budget.used).toBe(2); +}); + +test("gateway quota guard: an exhausted budget makes no Anthropic inference call", async () => { + await seed(2); + const config = configFor(() => answer(false)); + const budget = createRequestExecutionBudget({ maxTotalModelSends: 0, baseSendAllowance: 0, + finalRecoveryAllowance: 0, maxAlternateTargetSends: 0, maxTargetTransitions: 0 }); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", input: "Answer briefly", stream: false }), + }), config, { model: "", provider: "" }, { sendBudget: budget }); + await response.text(); + expect(sent).toHaveLength(0); + expect(budget.used).toBe(0); +}); + +test.each([400, 429])("gateway quota guard: continuation HTTP %i cannot refill a spent request budget", async status => { + await seed(3); + const config = configFor(body => sent.length === 1 + ? answer(body.stream === true, "0.11", "0.31", "I will modify the file now.") + : status === 429 ? limited() : Response.json({ type: "error", error: { + type: "invalid_request_error", message: "You're out of extra usage." } }, { status })); + const budget = createRequestExecutionBudget({ maxTotalModelSends: 2, baseSendAllowance: 2, + finalRecoveryAllowance: 0, maxAlternateTargetSends: 0, maxTargetTransitions: 0 }); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", stream: false, + input: "Please modify the file now", tools: [{ type: "function", name: "read_file", + description: "read a file", parameters: { type: "object" } }] }), + }), config, { model: "", provider: "" }, { sendBudget: budget }); + await response.text(); + expect(sent).toHaveLength(2); + expect(new Set(sent.map(row => row.authorization)).size).toBe(1); + expect(budget.used).toBe(2); +}); + +test("gateway quota guard: image sidecar recovery shares the inference ceiling", async () => { + await seed(3); + const config = configFor(() => limited()); + config.images = { bridgeEnabled: true }; + config.providers.xai = { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", + authMode: "key", apiKey: "synthetic-image-key" }; + const budget = createRequestExecutionBudget({ maxTotalModelSends: 1, baseSendAllowance: 1, + finalRecoveryAllowance: 0, maxAlternateTargetSends: 0, maxTargetTransitions: 0 }); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", stream: true, + input: "Answer briefly", tools: [{ type: "image_generation" }] }), + }), config, { model: "", provider: "" }, { sendBudget: budget }); + await response.text(); + expect(sent).toHaveLength(1); + expect(budget.used).toBe(1); +}); + function expectQuota(id: string, fiveHourPercent: number, weeklyPercent: number) { expect(getCachedProviderAccountQuota("anthropic", id)).toMatchObject({ fiveHourPercent, weeklyPercent }); } diff --git a/tests/adapters/anthropic/anthropic-replay-oauth-identity.test.ts b/tests/adapters/anthropic/anthropic-replay-oauth-identity.test.ts index 8fa19b1a3be..0f5645407bb 100644 --- a/tests/adapters/anthropic/anthropic-replay-oauth-identity.test.ts +++ b/tests/adapters/anthropic/anthropic-replay-oauth-identity.test.ts @@ -49,6 +49,20 @@ describe("OAuth source replay carries the Claude identity block (ANDREW-CLAUDE-4 expect((await replay(sdk, "oauth")).system).toEqual(sdk); }); + test("gateway cache guard: volatile tails never enter the stable cached system prefix", async () => { + const stable = [ + { type: "text", text: "Stable coding instructions." }, + { type: "text", text: "Stable project rules.", cache_control: { type: "ephemeral" } }, + ]; + const first = await replay([...stable, { type: "text", text: "turn: 1; modified: 1" }], "oauth"); + const second = await replay([...stable, { type: "text", text: "turn: 2; modified: 2" }], "oauth"); + expect(first.system.slice(0, 3)).toEqual([IDENTITY, ...stable]); + expect(JSON.stringify(first.system.slice(0, 3))).toBe(JSON.stringify(second.system.slice(0, 3))); + expect(first.system[3]).not.toEqual(second.system[3]); + expect(second.system[2].cache_control).toEqual({ type: "ephemeral" }); + expect(second.system[3].cache_control).toBeUndefined(); + }); + test("an API-key replay stays byte for byte", async () => { expect((await replay("You are Hermes Agent.")).system).toBe("You are Hermes Agent."); }); diff --git a/tests/server/agentradar-account-pin-e2e.test.ts b/tests/server/agentradar-account-pin-e2e.test.ts index 04a7b96bdca..0123a526ab5 100644 --- a/tests/server/agentradar-account-pin-e2e.test.ts +++ b/tests/server/agentradar-account-pin-e2e.test.ts @@ -17,6 +17,7 @@ type Seen = { path: string; auth: string | null; account: string | null; interna const seen: Seen[] = []; const limited = new Set(); const denied = new Set(); +const quotaSpent = new Set(); const enc = new TextEncoder(); const sse = (events: [string, object][]) => new Response(new ReadableStream({ start(c) { for (const [type, data] of events) c.enqueue(enc.encode(`event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`)); c.close(); }, @@ -36,6 +37,8 @@ const fake = Bun.serve({ seen.push({ path, auth, account: req.headers.get("chatgpt-account-id"), internal: [...req.headers.keys()].filter(k => k.startsWith("x-agentradar-")), search, image: body.includes('"type":"image"') }); } if (path === "/v1/messages") { + if (auth && quotaSpent.has(auth)) return Response.json({ type: "error", error: { + type: "invalid_request_error", message: "You're out of extra usage." } }, { status: 400 }); if (auth && limited.has(auth)) return rateLimited(retryAfter); if (auth && denied.has(auth)) return Response.json({ type: "error", error: { type: "oauth_org_not_allowed", message: "Your organization has disabled Claude subscription access for Claude Code. Use an Anthropic API key instead, or ask your admin to enable access." } }, { status: 403 }); @@ -146,11 +149,12 @@ test("outside an AgentRadar gateway a pin is refused, never served unpinned", () }); test("an AgentRadar account pin serves exactly its own stored account (pin.check)", async () => { - limited.clear(); denied.clear(); retryAfter = "30"; + limited.clear(); denied.clear(); quotaSpent.clear(); retryAfter = "30"; const ocx = await startOcx({ anthropic: { activeAccountId: "B", accounts: [ { id: "A", credential: cred("atA") }, { id: "B", credential: cred("atB") }, { id: "C", credential: cred("atC"), needsReauth: true }, - { id: "D", credential: cred("atD") }, { id: "E", credential: cred("atE") }] }, + { id: "D", credential: cred("atD") }, { id: "E", credential: cred("atE") }, + { id: "F", credential: cred("atF") }] }, xai: { activeAccountId: "gB", accounts: ["gA", "gB", "gC"].map(id => ({ id, credential: cred(`t${id}`) })) }, kimi: { activeAccountId: "kB", accounts: ["kA", "kB"].map(id => ({ id, credential: cred(`t${id}`) })) }, }, { @@ -239,6 +243,15 @@ test("an AgentRadar account pin serves exactly its own stored account (pin.check r = await msg(ocx); assert.equal(r.status, 200, r.text); assert.deepEqual(tokens(r, "/v1/messages"), ["Bearer atB"]); + // A newly rotatable extra-usage 400 must not spend a different account when a pin is present. + quotaSpent.add("Bearer atE"); + r = await msg(ocx, pin("claude", "E")); + assert.ok(r.status >= 400, r.text); + assert.deepEqual(tokens(r, "/v1/messages"), ["Bearer atE"]); + assert.match(r.text, /out of extra usage/); + assert.equal((await accounts(ocx, "anthropic")).activeAccountId, "B"); + quotaSpent.clear(); + // Claude credential denial on the pinned account: returned as is, never moved to E; the unpinned control does fail over. denied.add("Bearer atD"); r = await msg(ocx, pin("claude", "D")); @@ -290,7 +303,7 @@ test("an AgentRadar account pin serves exactly its own stored account (pin.check }, 120_000); test("pooled turns log and answer their served account hashed; one request walks every spent account (failover.check)", async () => { - limited.clear(); denied.clear(); retryAfter = "318747"; + limited.clear(); denied.clear(); quotaSpent.clear(); retryAfter = "318747"; const ids = ["A", "B", "C", "D", "E", "F", "G", "H"]; // The hosted gateway's pool defaults (AgentRadar images/ocx/pool-defaults.mjs). const ocx = await startOcx({ diff --git a/tests/server/terminal-guard-server.test.ts b/tests/server/terminal-guard-server.test.ts index e0256c0e4f6..385ec65d996 100644 --- a/tests/server/terminal-guard-server.test.ts +++ b/tests/server/terminal-guard-server.test.ts @@ -139,6 +139,7 @@ describe("server terminal guard integration", () => { const home = mkdtempSync(join(tmpdir(), "ocx-terminal-anthropic-pool-")); const previousHome = process.env.OPENCODEX_HOME; process.env.OPENCODEX_HOME = home; + takeInheritedSpendHome(); clearAnthropicAccountPoolState(); clearPoolRotationState(); try { @@ -218,6 +219,8 @@ describe("server terminal guard integration", () => { }); expect(logCtx.attempts?.[1]?.provider).not.toBe(logCtx.attempts?.[0]?.provider); } finally { + releaseInheritedSpendHome?.(); + releaseInheritedSpendHome = undefined; clearAnthropicAccountPoolState(); clearPoolRotationState(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; @@ -230,6 +233,7 @@ describe("server terminal guard integration", () => { const home = mkdtempSync(join(tmpdir(), "ocx-terminal-anthropic-denial-")); const previousHome = process.env.OPENCODEX_HOME; process.env.OPENCODEX_HOME = home; + takeInheritedSpendHome(); clearAnthropicAccountPoolState(); clearPoolRotationState(); try { @@ -340,6 +344,8 @@ describe("server terminal guard integration", () => { expect(auth.anthropic.accounts.find(account => account.id === "account-b")?.needsReauth).toBe(true); expect(auth.anthropic.accounts.find(account => account.id === "account-c")?.needsReauth).toBeUndefined(); } finally { + releaseInheritedSpendHome?.(); + releaseInheritedSpendHome = undefined; clearAnthropicAccountPoolState(); clearPoolRotationState(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME;