diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index 489c325d..a6e96bab 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -251,6 +251,13 @@ row per model-provider error the agent's own client reported, including errors i retried itself; filled by the codex adapter, empty on other backends), `crashed`, `crash_reason`. +`model_used` depends on the route. On a Bedrock route the Claude Code agent records the +resolved inference-profile id (e.g. `eu.anthropic.claude-sonnet-5`), and the SDK's alias +stays on each `messages[].model`. On every other route it is the id the agent reports. +`EvaluationResult.model_used` takes the last iteration's value, else the configured +model unresolved. Strip the region and `anthropic.` prefixes and any Bedrock version suffix +(`-v1:0`) before you group rows by model. + `record_kind` records producer provenance: `agent` for collected agent attempts and `standalone_user` for an unconsumed simulator utterance preserved by the dialog driver. Legacy records keep `null` when their provenance is unknown; grading never fills it by inference. diff --git a/docs/agents/CLAUDE_CODE.md b/docs/agents/CLAUDE_CODE.md index 4145e99b..81a79306 100644 --- a/docs/agents/CLAUDE_CODE.md +++ b/docs/agents/CLAUDE_CODE.md @@ -64,6 +64,11 @@ subprocess. `BEDROCK_MODEL` is the route-level default; an explicit `--model` / `-D agent.model=…` / task `agent.model` always wins over it, and bare aliases are auto-qualified to a regional inference profile (`eu.` / `us.` / `apac.` / `global.`). +On Bedrock, `model_used` (per iteration and on the task) records that resolved +inference-profile id, e.g. `eu.anthropic.claude-sonnet-5`. The SDK reports a friendly +alias (`claude-sonnet-5`) instead; that raw value stays on each `messages[].model`. +See [Report Schema → TurnRecord](../REPORT_SCHEMA.md#turnrecord) for the per-route contract. + > **For official benchmarking use the direct API** — the SDK reports accurate > token/cost there. See [User Guide → API Routing](../USER_GUIDE.md#api-routing--benchmarking). diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index 66236b06..76e43ca1 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -12,6 +12,16 @@ describe("resolvePricing", () => { ); }); + test("tolerates a Bedrock version suffix after region and vendor prefixes", () => { + expect( + resolvePricing("eu.anthropic.claude-sonnet-4-5-20250929-v1:0") + ?.outputPerMTok, + ).toBe(15); + expect(resolvePricing("eu.anthropic.claude-sonnet-4-6-v1:0")?.outputPerMTok).toBe( + 15, + ); + }); + test("returns null for unknown / missing models", () => { expect(resolvePricing("definitely-not-a-model")).toBeNull(); expect(resolvePricing(null)).toBeNull(); diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 876f131d..59a6784e 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -49,6 +49,8 @@ const _ROUTING_PREFIXES = [ "openrouter/", ]; const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."]; +// Bedrock's model version suffix ("…-20250929-v1:0"); never part of a rate key. +const _BEDROCK_VERSION_SUFFIX = /-v\d+:\d+$/; // Exported so the run header's model tally groups on the same key pricing looks // up on (lib/runs.ts::tallyModels). export function normalizeModel(model: string): string { @@ -66,7 +68,7 @@ export function normalizeModel(model: string): string { } } if (m.startsWith("anthropic.")) m = m.slice("anthropic.".length); - return m; + return m.replace(_BEDROCK_VERSION_SUFFIX, ""); } // Resolve pricing for a model id, tolerating routing/region prefixes and undated diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 7ae86cc1..0d87474e 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -714,6 +714,18 @@ def _finalize_token_usage(self) -> TokenUsage: self._agent._reprice_for_litellm(usage, self.effective_model) return usage + @property + def model_used(self) -> str | None: + """The model recorded on ``AgentEndEvent.model_used``: the resolved route model on Bedrock, else the SDK's. + + On Bedrock the SDK reports a friendly alias (``claude-sonnet-5``) for the + inference profile the run was pinned to (``eu.anthropic.claude-sonnet-5``). + That raw alias stays on each ``AssistantMessage.model``. + """ + if isinstance(self._agent.route, BedrockRoute) and self.effective_model: + return self.effective_model + return self.sdk_model_used + def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None: """Close orphaned tools + the open turn, emit the terminal AgentEndEvent, and on a crash build the partial TurnRecord. Idempotent.""" @@ -771,7 +783,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso iteration=self.iteration, user_input=self.user_input, agent_output=agent_output, - model_used=self.sdk_model_used, + model_used=self.model_used, assistant_turn_count=self.assistant_turn_count, messages=list(self.sdk_messages), num_turns=self.num_turns, diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index b02801bc..c8c1a893 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -347,7 +347,10 @@ class TurnRecord(BaseModel): default=None, description="Token usage for this turn (if available from agent SDK)" ) model_used: str | None = Field( - default=None, description="Model identifier used for this turn (e.g., 'claude-sonnet-4-5-20250514')" + default=None, + description="Model identifier used for this turn (e.g., 'claude-sonnet-4-5-20250514'). On a Bedrock " + + "route the Claude Code agent records the resolved inference-profile id (e.g., " + + "'eu.anthropic.claude-sonnet-5'), not the SDK's alias, so normalize the id before grouping across routes.", ) assistant_turn_count: int = Field( default=0, @@ -550,7 +553,10 @@ class EvaluationResult(BaseModel): variant_id: str = Field(default="default", description="ID of the experiment variant") agent_type: str = Field(description="Type of agent used (registered kind string, e.g. 'claude-code')") model_used: str | None = Field( - default=None, description="Model identifier used for the evaluation (resolved from iterations or agent config)" + default=None, + description="Model identifier used for the evaluation: the last iteration's TurnRecord.model_used, else " + + "the configured agent model. The config fallback is not route-resolved, so a Bedrock row with no " + + "recorded iteration carries the bare configured alias.", ) # Execution metadata diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index 0b9b1c3e..13e3096f 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -13,6 +13,7 @@ generated file. """ +import re from collections.abc import Iterable, Mapping from dataclasses import dataclass from types import MappingProxyType @@ -213,12 +214,14 @@ def register_pricing(rates: dict[str, ModelPricing]) -> None: # Mirrors models.routing._BEDROCK_KNOWN_PREFIXES. A Bedrock route qualifies a bare # alias; the table is keyed on the bare alias, so strip these before the lookup. _BEDROCK_REGION_PREFIXES: tuple[str, ...] = ("eu.", "us.", "apac.", "global.") +_BEDROCK_VERSION_SUFFIX = re.compile(r"-v\d+:\d+$") def _normalize_model(model: str) -> str: - """Strip Bedrock region + vendor qualifiers back to the bare pricing key. + """Strip Bedrock region + vendor qualifiers and version suffix back to the bare pricing key. - ``eu.anthropic.claude-opus-4-8`` → ``claude-opus-4-8``. Idempotent on + ``eu.anthropic.claude-opus-4-8`` → ``claude-opus-4-8``; + ``eu.anthropic.claude-sonnet-4-5-20250929-v1:0`` → ``claude-sonnet-4-5-20250929``. Idempotent on already-bare aliases (``claude-opus-4-8`` / ``gpt-5-codex`` pass through), so it is safe to apply unconditionally for every route. """ @@ -237,7 +240,7 @@ def _normalize_model(model: str) -> str: break if model.startswith("anthropic."): model = model[len("anthropic.") :] - return model + return _BEDROCK_VERSION_SUFFIX.sub("", model) def is_priced(model: str) -> bool: diff --git a/tests/test_agent.py b/tests/test_agent.py index 03799fae..15a86f78 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -12,7 +12,7 @@ from coder_eval.agent import AgentState from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.errors import AgentCrashError, TurnTimeoutError -from coder_eval.models import AgentKind, parse_agent_config +from coder_eval.models import AgentKind, ApiRoute, BedrockRoute, DirectRoute, LiteLLMRoute, parse_agent_config def test_claude_agent_initialization(): @@ -2032,6 +2032,57 @@ def test_on_user_message_resolves_pending_command(self): assert len(tool_ends) == 1 assert tool_ends[0].status == ToolEndStatus.OK + def _run_alias_turn(self, route: ApiRoute, config_model: str | None, *, crashed: bool = False): + """Drive one turn whose SDK reports the alias ``claude-sonnet-5``; a crash ends it as a TIMEOUT.""" + from coder_eval.streaming.events import AgentEndStatus + from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage, TextBlock + + agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE), route=route) + state, events = self._state(agent) + state.effective_model = agent._resolve_effective_model(config_model, {}, None) + + state.on_assistant_message(AssistantMessage([TextBlock("done")], message_id="m1", model="claude-sonnet-5")) + status = AgentEndStatus.TIMEOUT if crashed else AgentEndStatus.COMPLETED + state.finalize(status, crashed=crashed, crash_reason="boom" if crashed else None) + return agent, state, events + + @pytest.mark.parametrize( + ("route", "config_model", "expected"), + [ + pytest.param( + BedrockRoute(region="eu-north-1"), + "claude-sonnet-5", + "eu.anthropic.claude-sonnet-5", + id="bedrock-records-qualified-alias", + ), + pytest.param( + BedrockRoute(region="eu-north-1"), + None, + "claude-sonnet-5", + id="bedrock-without-resolved-model-falls-back-to-sdk", + ), + pytest.param(DirectRoute(), "sonnet", "claude-sonnet-5", id="direct-records-sdk-model"), + pytest.param( + LiteLLMRoute(model="zai.glm-5"), "zai.glm-5", "claude-sonnet-5", id="litellm-records-sdk-model" + ), + ], + ) + def test_model_used_on_agent_end(self, route: ApiRoute, config_model: str | None, expected: str): + from coder_eval.streaming.events import AgentEndEvent + + _, state, events = self._run_alias_turn(route, config_model) + + (end,) = [e for e in events if isinstance(e, AgentEndEvent)] + assert end.model_used == expected + assert state.collector.build_turn_record().model_used == expected + assert state.sdk_messages[0].model == "claude-sonnet-5" + + def test_model_used_on_crashed_partial_turn(self): + agent, _, _ = self._run_alias_turn(BedrockRoute(region="eu-north-1"), "claude-sonnet-5", crashed=True) + + assert agent.pending_turn is not None + assert agent.pending_turn.model_used == "eu.anthropic.claude-sonnet-5" + class TestAgentBaseHelpers: """Unit tests for the shared mid-turn failure kernels on the Agent base.""" diff --git a/tests/test_pricing_registry.py b/tests/test_pricing_registry.py index b9e0123f..ecae0afc 100644 --- a/tests/test_pricing_registry.py +++ b/tests/test_pricing_registry.py @@ -29,6 +29,20 @@ def test_registered_bare_key_resolves_through_prefixed_lookup(): assert calculate_cost("eu.acme-1", 1_000_000, 0) == 1.0 +@pytest.mark.parametrize( + ("model", "bare"), + [ + ("eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "claude-sonnet-4-5-20250929"), + ("us.anthropic.claude-sonnet-4-6-v1:0", "claude-sonnet-4-6"), + ("anthropic.claude-sonnet-4-5-20250929-v2:1", "claude-sonnet-4-5-20250929"), + ], +) +def test_bedrock_version_suffix_prices_as_bare_key(model: str, bare: str): + """Claude Code on Bedrock records the configured id, e.g. ``.env.example``'s ``…-v1:0``, as ``model_used``.""" + assert pricing._normalize_model(model) == bare + assert calculate_cost(model, 1_000_000, 1_000_000) == calculate_cost(bare, 1_000_000, 1_000_000) > 0 + + def test_openrouter_provider_prefix_normalizes_to_bare_key(): """An agent that addresses OpenRouter natively (OpenCode) reports the model WITH its provider prefix, while the rate-card keys are the bare vendor/model