Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/REPORT_SCHEMA.md
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,13 @@ row per model-provider error the agent's own client reported, including errors i
retried itself; filled by the codex adapter, empty on other backends), `crashed`,
`crash_reason`.

`model_used` depends on the route. On a Bedrock route the Claude Code agent records the
resolved inference-profile id (e.g. `eu.anthropic.claude-sonnet-5`), and the SDK's alias
stays on each `messages[].model`. On every other route it is the id the agent reports.
`EvaluationResult.model_used` takes the last iteration's value, else the configured
model unresolved. Strip the region and `anthropic.` prefixes and any Bedrock version suffix
(`-v1:0`) before you group rows by model.

`record_kind` records producer provenance: `agent` for collected agent attempts and
`standalone_user` for an unconsumed simulator utterance preserved by the dialog driver.
Legacy records keep `null` when their provenance is unknown; grading never fills it by inference.
Expand Down
5 changes: 5 additions & 0 deletions docs/agents/CLAUDE_CODE.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ subprocess. `BEDROCK_MODEL` is the route-level default; an explicit `--model` /
`-D agent.model=…` / task `agent.model` always wins over it, and bare aliases are
auto-qualified to a regional inference profile (`eu.` / `us.` / `apac.` / `global.`).

On Bedrock, `model_used` (per iteration and on the task) records that resolved
inference-profile id, e.g. `eu.anthropic.claude-sonnet-5`. The SDK reports a friendly
alias (`claude-sonnet-5`) instead; that raw value stays on each `messages[].model`.
See [Report Schema → TurnRecord](../REPORT_SCHEMA.md#turnrecord) for the per-route contract.

> **For official benchmarking use the direct API** — the SDK reports accurate
> token/cost there. See [User Guide → API Routing](../USER_GUIDE.md#api-routing--benchmarking).

Expand Down
10 changes: 10 additions & 0 deletions evalboard/lib/__tests__/pricing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@ describe("resolvePricing", () => {
);
});

test("tolerates a Bedrock version suffix after region and vendor prefixes", () => {
expect(
resolvePricing("eu.anthropic.claude-sonnet-4-5-20250929-v1:0")
?.outputPerMTok,
).toBe(15);
expect(resolvePricing("eu.anthropic.claude-sonnet-4-6-v1:0")?.outputPerMTok).toBe(
15,
);
});

test("returns null for unknown / missing models", () => {
expect(resolvePricing("definitely-not-a-model")).toBeNull();
expect(resolvePricing(null)).toBeNull();
Expand Down
4 changes: 3 additions & 1 deletion evalboard/lib/pricing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ const _ROUTING_PREFIXES = [
"openrouter/",
];
const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."];
// Bedrock's model version suffix ("…-20250929-v1:0"); never part of a rate key.
const _BEDROCK_VERSION_SUFFIX = /-v\d+:\d+$/;
// Exported so the run header's model tally groups on the same key pricing looks
// up on (lib/runs.ts::tallyModels).
export function normalizeModel(model: string): string {
Expand All @@ -66,7 +68,7 @@ export function normalizeModel(model: string): string {
}
}
if (m.startsWith("anthropic.")) m = m.slice("anthropic.".length);
return m;
return m.replace(_BEDROCK_VERSION_SUFFIX, "");
}

// Resolve pricing for a model id, tolerating routing/region prefixes and undated
Expand Down
14 changes: 13 additions & 1 deletion src/coder_eval/agents/claude_code_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,18 @@ def _finalize_token_usage(self) -> TokenUsage:
self._agent._reprice_for_litellm(usage, self.effective_model)
return usage

@property
def model_used(self) -> str | None:
"""The model recorded on ``AgentEndEvent.model_used``: the resolved route model on Bedrock, else the SDK's.

On Bedrock the SDK reports a friendly alias (``claude-sonnet-5``) for the
inference profile the run was pinned to (``eu.anthropic.claude-sonnet-5``).
That raw alias stays on each ``AssistantMessage.model``.
"""
if isinstance(self._agent.route, BedrockRoute) and self.effective_model:
return self.effective_model
return self.sdk_model_used

def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None:
"""Close orphaned tools + the open turn, emit the terminal AgentEndEvent,
and on a crash build the partial TurnRecord. Idempotent."""
Expand Down Expand Up @@ -771,7 +783,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso
iteration=self.iteration,
user_input=self.user_input,
agent_output=agent_output,
model_used=self.sdk_model_used,
model_used=self.model_used,
assistant_turn_count=self.assistant_turn_count,
messages=list(self.sdk_messages),
num_turns=self.num_turns,
Expand Down
10 changes: 8 additions & 2 deletions src/coder_eval/models/results.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,10 @@ class TurnRecord(BaseModel):
default=None, description="Token usage for this turn (if available from agent SDK)"
)
model_used: str | None = Field(
default=None, description="Model identifier used for this turn (e.g., 'claude-sonnet-4-5-20250514')"
default=None,
description="Model identifier used for this turn (e.g., 'claude-sonnet-4-5-20250514'). On a Bedrock "
+ "route the Claude Code agent records the resolved inference-profile id (e.g., "
+ "'eu.anthropic.claude-sonnet-5'), not the SDK's alias, so normalize the id before grouping across routes.",
)
assistant_turn_count: int = Field(
default=0,
Expand Down Expand Up @@ -550,7 +553,10 @@ class EvaluationResult(BaseModel):
variant_id: str = Field(default="default", description="ID of the experiment variant")
agent_type: str = Field(description="Type of agent used (registered kind string, e.g. 'claude-code')")
model_used: str | None = Field(
default=None, description="Model identifier used for the evaluation (resolved from iterations or agent config)"
default=None,
description="Model identifier used for the evaluation: the last iteration's TurnRecord.model_used, else "
+ "the configured agent model. The config fallback is not route-resolved, so a Bedrock row with no "
+ "recorded iteration carries the bare configured alias.",
)

# Execution metadata
Expand Down
9 changes: 6 additions & 3 deletions src/coder_eval/pricing.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
generated file.
"""

import re
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from types import MappingProxyType
Expand Down Expand Up @@ -213,12 +214,14 @@ def register_pricing(rates: dict[str, ModelPricing]) -> None:
# Mirrors models.routing._BEDROCK_KNOWN_PREFIXES. A Bedrock route qualifies a bare
# alias; the table is keyed on the bare alias, so strip these before the lookup.
_BEDROCK_REGION_PREFIXES: tuple[str, ...] = ("eu.", "us.", "apac.", "global.")
_BEDROCK_VERSION_SUFFIX = re.compile(r"-v\d+:\d+$")


def _normalize_model(model: str) -> str:
"""Strip Bedrock region + vendor qualifiers back to the bare pricing key.
"""Strip Bedrock region + vendor qualifiers and version suffix back to the bare pricing key.

``eu.anthropic.claude-opus-4-8`` → ``claude-opus-4-8``. Idempotent on
``eu.anthropic.claude-opus-4-8`` → ``claude-opus-4-8``;
``eu.anthropic.claude-sonnet-4-5-20250929-v1:0`` → ``claude-sonnet-4-5-20250929``. Idempotent on
already-bare aliases (``claude-opus-4-8`` / ``gpt-5-codex`` pass through),
so it is safe to apply unconditionally for every route.
"""
Expand All @@ -237,7 +240,7 @@ def _normalize_model(model: str) -> str:
break
if model.startswith("anthropic."):
model = model[len("anthropic.") :]
return model
return _BEDROCK_VERSION_SUFFIX.sub("", model)


def is_priced(model: str) -> bool:
Expand Down
53 changes: 52 additions & 1 deletion tests/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from coder_eval.agent import AgentState
from coder_eval.agents.claude_code_agent import ClaudeCodeAgent
from coder_eval.errors import AgentCrashError, TurnTimeoutError
from coder_eval.models import AgentKind, parse_agent_config
from coder_eval.models import AgentKind, ApiRoute, BedrockRoute, DirectRoute, LiteLLMRoute, parse_agent_config


def test_claude_agent_initialization():
Expand Down Expand Up @@ -2032,6 +2032,57 @@ def test_on_user_message_resolves_pending_command(self):
assert len(tool_ends) == 1
assert tool_ends[0].status == ToolEndStatus.OK

def _run_alias_turn(self, route: ApiRoute, config_model: str | None, *, crashed: bool = False):
"""Drive one turn whose SDK reports the alias ``claude-sonnet-5``; a crash ends it as a TIMEOUT."""
from coder_eval.streaming.events import AgentEndStatus
from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage, TextBlock

agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE), route=route)
state, events = self._state(agent)
state.effective_model = agent._resolve_effective_model(config_model, {}, None)

state.on_assistant_message(AssistantMessage([TextBlock("done")], message_id="m1", model="claude-sonnet-5"))
status = AgentEndStatus.TIMEOUT if crashed else AgentEndStatus.COMPLETED
state.finalize(status, crashed=crashed, crash_reason="boom" if crashed else None)
return agent, state, events

@pytest.mark.parametrize(
("route", "config_model", "expected"),
[
pytest.param(
BedrockRoute(region="eu-north-1"),
"claude-sonnet-5",
"eu.anthropic.claude-sonnet-5",
id="bedrock-records-qualified-alias",
),
pytest.param(
BedrockRoute(region="eu-north-1"),
None,
"claude-sonnet-5",
id="bedrock-without-resolved-model-falls-back-to-sdk",
),
pytest.param(DirectRoute(), "sonnet", "claude-sonnet-5", id="direct-records-sdk-model"),
pytest.param(
LiteLLMRoute(model="zai.glm-5"), "zai.glm-5", "claude-sonnet-5", id="litellm-records-sdk-model"
),
],
)
def test_model_used_on_agent_end(self, route: ApiRoute, config_model: str | None, expected: str):
from coder_eval.streaming.events import AgentEndEvent

_, state, events = self._run_alias_turn(route, config_model)

(end,) = [e for e in events if isinstance(e, AgentEndEvent)]
assert end.model_used == expected
assert state.collector.build_turn_record().model_used == expected
assert state.sdk_messages[0].model == "claude-sonnet-5"

def test_model_used_on_crashed_partial_turn(self):
agent, _, _ = self._run_alias_turn(BedrockRoute(region="eu-north-1"), "claude-sonnet-5", crashed=True)

assert agent.pending_turn is not None
assert agent.pending_turn.model_used == "eu.anthropic.claude-sonnet-5"


class TestAgentBaseHelpers:
"""Unit tests for the shared mid-turn failure kernels on the Agent base."""
Expand Down
14 changes: 14 additions & 0 deletions tests/test_pricing_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,20 @@ def test_registered_bare_key_resolves_through_prefixed_lookup():
assert calculate_cost("eu.acme-1", 1_000_000, 0) == 1.0


@pytest.mark.parametrize(
("model", "bare"),
[
("eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "claude-sonnet-4-5-20250929"),
("us.anthropic.claude-sonnet-4-6-v1:0", "claude-sonnet-4-6"),
("anthropic.claude-sonnet-4-5-20250929-v2:1", "claude-sonnet-4-5-20250929"),
],
)
def test_bedrock_version_suffix_prices_as_bare_key(model: str, bare: str):
"""Claude Code on Bedrock records the configured id, e.g. ``.env.example``'s ``…-v1:0``, as ``model_used``."""
assert pricing._normalize_model(model) == bare
assert calculate_cost(model, 1_000_000, 1_000_000) == calculate_cost(bare, 1_000_000, 1_000_000) > 0


def test_openrouter_provider_prefix_normalizes_to_bare_key():
"""An agent that addresses OpenRouter natively (OpenCode) reports the model
WITH its provider prefix, while the rate-card keys are the bare vendor/model
Expand Down
Loading