diff --git a/docs/AB_EXPERIMENTS.md b/docs/AB_EXPERIMENTS.md index 76b1551a..c20fa2c6 100644 --- a/docs/AB_EXPERIMENTS.md +++ b/docs/AB_EXPERIMENTS.md @@ -412,6 +412,11 @@ The reports (generated by `reports/experiment.py`) are: duration, total tokens, and — when `repeats > 1` — `per_replicate_scores` so you can see variance, not just a point estimate). `experiment.html` renders the same data for browsing. + The Aggregate Metrics table has a **Model** row: the models each variant ran on. + When the variants ran more than one model, the turns row reads **Assistant + Turns (cross-model)**, it has no p-value, and a note under the table says why: + a turn count depends on how much work a model batches into one call, so a turn + difference across models is mostly model style, not variant cost. - **`/variant.md` / `variant.json`** — per-variant rollup (plus `variant.html`). - **`run.md` / `run.json`** (run root) — the flat batch log across all diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index c1e11220..07ea4bc9 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -305,7 +305,10 @@ The cross-variant summary: `{task_id, variant_results, best_variant, is_tie, score_spread, replicate_count}`, where each `VariantResult` carries `{variant_id, task_id, weighted_score, final_status, duration_seconds, total_tokens, iteration_count, - total_assistant_turns, reference_similarity, replicate_index, replicate_count}`. + total_assistant_turns, models_used, reference_similarity, replicate_index, + replicate_count}`. `models_used` is the sorted list of distinct `model_used` + values across the replicates (`[]` when none recorded a model, and in an + `experiment.json` written before the field existed). `weighted_score` is `float | None`: `null` only when every replicate was **ungraded**. An **errored** replicate counts as `0.0` — same rule as `VariantAggregate.average_score` above. That is deliberate: dropping errored diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 8c948156..d58416f2 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -363,6 +363,14 @@ The count compared against the budget is **visible turns** — one per tool call plus one for the agent's final reply. It is *not* `total_turns`, which counts model API calls (the `max_turns` unit), and one call can batch several tool calls. +Turn counts are comparable only **within one model**. Models differ in how much +work they batch into one call (one model chains a whole phase into a single +`Bash` call, another sends one probe per call), so the same task and prompt can +take twice the turns on a different model. Compare turns between runs or variants +on the same model. Cost and wall-clock time stay comparable across models. The +experiment report shows the model of each variant and marks a cross-model turn +comparison (see [Reading the Report](AB_EXPERIMENTS.md#reading-the-report)). + Set it to the number of turns a competent agent should need for the task. Pick budgets consistently across a suite — the headline % is only comparable when tasks are measured against realistic, like-for-like targets. Omit it (the diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 59a6784e..b2bf79b1 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -34,10 +34,10 @@ export interface Pricing { export { PRICING }; // Strip the LiteLLM/Bedrock routing + region/vendor prefixes back to the bare -// pricing key — mirror of src/coder_eval/pricing.py::_normalize_model, since the +// pricing key — mirror of src/coder_eval/pricing.py::normalize_model, since the // recorded model_used arrives qualified (e.g. "converse/zai.glm-5", // "eu.anthropic.claude-sonnet-4-6"). Idempotent on already-bare ids. -// "openrouter/" is here for the same reason it is in _normalize_model: a harness +// "openrouter/" is here for the same reason it is in normalize_model: a harness // that addresses OpenRouter natively (OpenCode) records the model WITH its // provider prefix ("openrouter/deepseek/deepseek-v4-pro"), while the rate keys // are the bare vendor/model ids the LiteLLM route records. Without the strip the diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index ece7d444..62cb0c6b 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -198,6 +198,13 @@ class VariantResult(BaseModel): # noqa: CE009 -- persisted result model; round- total_tokens: int | None = None iteration_count: int | None = None total_assistant_turns: int | None = None + models_used: list[str] = Field( + default_factory=list, + description=( + "Distinct `model_used` values across the folded replicates, sorted. Empty when no replicate " + "recorded a model. The experiment report reads it to mark a cross-model turn comparison." + ), + ) reference_similarity: float | None = None replicate_index: int = Field( default=0, diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index f93b84d2..f53377f9 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -845,6 +845,7 @@ def _fold_replicates(task_id: str, variant_id: str, reps: list[TaskResult]) -> V total_tokens=sum(token_vals) if token_vals else None, iteration_count=round(sum(iter_counts) / len(iter_counts)) if iter_counts else None, total_assistant_turns=round(sum(asst_turns) / len(asst_turns)) if asst_turns else None, + models_used=sorted({r.result.model_used for r in reps if r.result.model_used}), reference_similarity=_mean_reference_similarity(reps), replicate_index=0, # aggregate — points at first replicate for link rendering replicate_count=len(reps), diff --git a/src/coder_eval/pricing.py b/src/coder_eval/pricing.py index 13e3096f..d35014a2 100644 --- a/src/coder_eval/pricing.py +++ b/src/coder_eval/pricing.py @@ -217,7 +217,7 @@ def register_pricing(rates: dict[str, ModelPricing]) -> None: _BEDROCK_VERSION_SUFFIX = re.compile(r"-v\d+:\d+$") -def _normalize_model(model: str) -> str: +def normalize_model(model: str) -> str: """Strip Bedrock region + vendor qualifiers and version suffix back to the bare pricing key. ``eu.anthropic.claude-opus-4-8`` → ``claude-opus-4-8``; @@ -245,7 +245,7 @@ def _normalize_model(model: str) -> str: def is_priced(model: str) -> bool: """Whether the rate card can price this model (after prefix normalization).""" - return _lookup_rate(_normalize_model(model)) is not None + return _lookup_rate(normalize_model(model)) is not None def unpriced_models(models: Iterable[str | None]) -> list[str]: @@ -275,7 +275,7 @@ def calculate_cost( the same as the bare ``claude-opus-4-8``. Returns None if the (normalized) model is not in the pricing table. """ - pricing = _lookup_rate(_normalize_model(model)) + pricing = _lookup_rate(normalize_model(model)) if pricing is None: return None diff --git a/src/coder_eval/reports/experiment.py b/src/coder_eval/reports/experiment.py index 4501c92a..12ea5b97 100644 --- a/src/coder_eval/reports/experiment.py +++ b/src/coder_eval/reports/experiment.py @@ -14,6 +14,7 @@ from ..run_record import eval_result_to_task_dict from ..stats import bootstrap_mean_ci, stddev, welch_t_test, wilson_interval from .helpers import ( + CROSS_MODEL_TURNS_NOTE, VariantSeries, collect_variant_series, describe_prompt_config, @@ -23,6 +24,8 @@ is_env_table_key, load_variant_eval_results, paired_comparison, + turns_cross_model, + variant_models, ) from .html import write_experiment_html, write_variant_html from .markdown import ReportGenerator, resolve_agent_settings, resolve_context_window @@ -189,10 +192,13 @@ def _aggregate_stat_rows( show_p_values: bool, vid_a: str, vid_b: str, + *, + cross_model: bool, ) -> list[str]: """The mean ± stddev rows of the Aggregate Metrics table (Score, Avg Duration, and the optional Assistant Turns / Tokens rows), each with a Welch t-test - p-value when ``show_p_values``.""" + p-value when ``show_p_values``. When ``cross_model``, the Assistant Turns row + is marked and has no p-value.""" lines: list[str] = [] # Row: Score (mean ± stddev, p-value) @@ -215,11 +221,11 @@ def _aggregate_stat_rows( # Row: Assistant Turns (if data available) if any(series[vid].asst_turns for vid in result.variant_ids): - row = "| Assistant Turns" + row = "| Assistant Turns (cross-model)" if cross_model else "| Assistant Turns" for vid in result.variant_ids: row += f" | {fmt_mean_sd(series[vid].asst_turns, '.1f')}" if show_p_values: - p = welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns) + p = None if cross_model else welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns) row += f" | {fmt_p(p)}" lines.append(row + " |") @@ -256,8 +262,23 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]: sep += "|" lines = ["", "## Aggregate Metrics", "", header, sep] + + # Row: Model (if any variant recorded one). Turns depend on the model, so + # the table that shows them also shows what each variant ran on. + models = variant_models(result) + cross_model = turns_cross_model(models) + if any(models.values()): + row = "| Model" + for vid in result.variant_ids: + row += f" | {', '.join(models[vid]) or 'n/a'}" + if show_p_values: + row += " | —" + lines.append(row + " |") + lines += ExperimentReportGenerator._aggregate_count_rows(result, show_p_values) - lines += ExperimentReportGenerator._aggregate_stat_rows(result, series, show_p_values, vid_a, vid_b) + lines += ExperimentReportGenerator._aggregate_stat_rows( + result, series, show_p_values, vid_a, vid_b, cross_model=cross_model + ) # Row: Replicates/task (if any variant ran >1 replicate) if any(result.variant_aggregates[vid].replicate_count > 1 for vid in result.variant_ids): @@ -269,6 +290,9 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]: row += " | —" lines.append(row + " |") + if cross_model and any(series[vid].asst_turns for vid in result.variant_ids): + lines += ["", f"*{CROSS_MODEL_TURNS_NOTE}*"] + return lines @staticmethod diff --git a/src/coder_eval/reports/helpers.py b/src/coder_eval/reports/helpers.py index bdcb1189..7c6d7e35 100644 --- a/src/coder_eval/reports/helpers.py +++ b/src/coder_eval/reports/helpers.py @@ -16,6 +16,7 @@ from __future__ import annotations import logging +import re from pathlib import Path from typing import NamedTuple @@ -26,6 +27,7 @@ TaskExperimentSummary, ) from ..path_utils import TASK_JSON_FILENAME +from ..pricing import normalize_model from ..stats import cohens_d, mean, paired_t_ci, paired_t_test, stddev @@ -126,6 +128,54 @@ def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries] return series +# Shown under an Aggregate Metrics table whose Assistant Turns row compares more +# than one model. Shared so the markdown and HTML reports say the same thing. +CROSS_MODEL_TURNS_NOTE = ( + "These turn counts come from more than one model. How many turns a task takes depends on how much" + + " work a model batches into one call, so a turn difference across models is mostly model style," + + " not variant cost. Compare turns only between variants on the same model." +) + + +# A dated snapshot id (`claude-haiku-4-5-20251001`) is the same model as its alias +# (`claude-haiku-4-5`). A row that replied records the resolved snapshot, a row that +# crashed first keeps the configured alias. +_SNAPSHOT_DATE_SUFFIX = re.compile(r"-20\d{6}$") + + +# Claude Code stamps this on an assistant message it makes up itself (an API +# error it reports without a model call), and a row that ends on one records it +# as `model_used`. It names no model, so it never counts as one. +_SYNTHETIC_MODEL = "" + + +def variant_models(result: ExperimentResult) -> dict[str, list[str]]: + """The distinct models each variant ran on, sorted, keyed by variant id. + + Ids are normalized with ``pricing.normalize_model`` and a trailing snapshot date + is dropped: a row that errored before the model resolved keeps the qualified route + id (``eu.anthropic.claude-sonnet-5``) or the configured alias, and a raw comparison + would read either as a second model. + """ + models: dict[str, set[str]] = {vid: set() for vid in result.variant_ids} + for ts in result.task_summaries: + for vr in ts.variant_results: + if vr.variant_id in models: + models[vr.variant_id].update( + _SNAPSHOT_DATE_SUFFIX.sub("", normalize_model(m)) for m in vr.models_used if m != _SYNTHETIC_MODEL + ) + return {vid: sorted(found) for vid, found in models.items()} + + +def turns_cross_model(models: dict[str, list[str]]) -> bool: + """Whether the variants' turn counts come from more than one model. + + A variant with no recorded model is unknown, not different, so it never + makes the comparison cross-model on its own. + """ + return len({m for found in models.values() for m in found}) > 1 + + class PairedComparison(NamedTuple): """A 2-variant paired comparison over per-task mean scores. diff --git a/src/coder_eval/reports/html.py b/src/coder_eval/reports/html.py index 03b1a1d7..545428e7 100644 --- a/src/coder_eval/reports/html.py +++ b/src/coder_eval/reports/html.py @@ -27,6 +27,7 @@ from ..result_metrics import agent_wall_ms, expected_turns_overage, turn_time_buckets from ..stats import stddev, welch_t_test from .helpers import ( + CROSS_MODEL_TURNS_NOTE, collect_variant_series, describe_prompt_config, fmt_mean_sd, @@ -35,6 +36,8 @@ is_env_table_key, load_variant_eval_results, paired_comparison, + turns_cross_model, + variant_models, ) from .markdown import ( SLOW_PARAMS_PREVIEW_CHARS, @@ -1386,6 +1389,12 @@ def _row(label: str, values: list[str], p: str | None) -> str: return "" + "".join(cells) + "" rows: list[str] = [] + # Turns depend on the model, so the table that shows them also shows what + # each variant ran on. + models = variant_models(result) + cross_model = turns_cross_model(models) + if any(models.values()): + rows.append(_row("Model", [", ".join(models[vid]) or "n/a" for vid in result.variant_ids], None)) rows.append(_row("Tasks Run", [str(result.variant_aggregates[vid].tasks_run) for vid in result.variant_ids], None)) rows.append( _row( @@ -1427,12 +1436,15 @@ def _pass_rate(vid: str) -> str: fmt_p(welch_t_test(series[vid_a].durations, series[vid_b].durations)) if show_p else None, ) ) - if any(series[vid].asst_turns for vid in result.variant_ids): + has_turns = any(series[vid].asst_turns for vid in result.variant_ids) + if has_turns: rows.append( _row( - "Assistant Turns", + "Assistant Turns (cross-model)" if cross_model else "Assistant Turns", [fmt_mean_sd(series[vid].asst_turns, ".1f") for vid in result.variant_ids], - fmt_p(welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)) if show_p else None, + fmt_p(welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)) + if show_p and not cross_model + else None, ) ) if any(series[vid].tokens for vid in result.variant_ids): @@ -1444,6 +1456,7 @@ def _pass_rate(vid: str) -> str: ) ) + note = f'\n

{_esc(CROSS_MODEL_TURNS_NOTE)}

' if cross_model and has_turns else "" return f"""

Aggregate Metrics

@@ -1451,7 +1464,7 @@ def _pass_rate(vid: str) -> str: {header_html} {"".join(rows)} -
+{note} """ diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index cde2567f..70aa167d 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -1,6 +1,7 @@ """Tests for experiment result models and report generation.""" import json +from html import escape as html_escape import pytest @@ -15,6 +16,7 @@ VariantResult, ) from coder_eval.reports import ExperimentReportGenerator, describe_prompt_config +from coder_eval.reports.helpers import CROSS_MODEL_TURNS_NOTE from tests._fixtures.report_snapshots import assert_matches_snapshot @@ -1293,6 +1295,170 @@ def test_result_for_unknown_variant_is_ignored(self): assert set(series) == {"a"} +class TestCrossModelTurns: + """Turn counts depend on the model, so a cross-model turn comparison is marked (#98).""" + + @staticmethod + def _make_result(models: dict[str, list[str]]) -> ExperimentResult: + """Two tasks; each variant's turns differ enough that a Welch test reports a p-value.""" + variant_ids = list(models) + turns = {vid: [5 + 20 * i, 6 + 20 * i] for i, vid in enumerate(variant_ids)} + return ExperimentResult( + experiment_id="exp", + description="d", + variant_ids=variant_ids, + task_summaries=[ + TaskExperimentSummary( + task_id=task_id, + variant_results=[ + VariantResult( + variant_id=vid, + task_id=task_id, + weighted_score=1.0, + final_status="SUCCESS", + duration_seconds=10.0, + total_assistant_turns=turns[vid][t], + models_used=models[vid], + ) + for vid in variant_ids + ], + best_variant=variant_ids[0], + is_tie=True, + score_spread=0.0, + ) + for t, task_id in enumerate(["t1", "t2"]) + ], + variant_aggregates={ + vid: VariantAggregate( + variant_id=vid, + tasks_run=2, + tasks_succeeded=2, + tasks_failed=0, + tasks_error=0, + average_score=1.0, + average_duration=10.0, + ) + for vid in variant_ids + }, + total_duration_seconds=40.0, + ) + + @staticmethod + def _row(md: str, label: str) -> str: + return next(line for line in md.splitlines() if line.startswith(f"| {label} |")) + + def test_same_model_keeps_the_turns_p_value(self): + md = ExperimentReportGenerator.generate_experiment_report( + self._make_result({"a": ["claude-sonnet-5"], "b": ["claude-sonnet-5"]}) + ) + assert self._row(md, "Model") == "| Model | claude-sonnet-5 | claude-sonnet-5 | — |" + turns = self._row(md, "Assistant Turns") + assert not turns.endswith("| — |") + assert "cross-model" not in md + assert CROSS_MODEL_TURNS_NOTE not in md + + def test_different_models_mark_turns_and_drop_their_p_value(self): + md = ExperimentReportGenerator.generate_experiment_report( + self._make_result({"terra": ["gpt-5.6-terra"], "sonnet": ["claude-sonnet-5"]}) + ) + assert self._row(md, "Model") == "| Model | gpt-5.6-terra | claude-sonnet-5 | — |" + assert ( + self._row(md, "Assistant Turns (cross-model)") + == "| Assistant Turns (cross-model) | 5.5 ± 0.7 | 25.5 ± 0.7 | — |" + ) + assert f"*{CROSS_MODEL_TURNS_NOTE}*" in md + # Cost and wall-clock stay comparable across models: only the turns lose their p-value. + assert not self._row(md, "Avg Duration (s)").endswith("| — |") + + def test_route_qualified_id_is_the_same_model(self): + """An errored row keeps the qualified route id; it is not a second model.""" + md = ExperimentReportGenerator.generate_experiment_report( + self._make_result({"a": ["claude-sonnet-5", "eu.anthropic.claude-sonnet-5"], "b": ["claude-sonnet-5"]}) + ) + assert self._row(md, "Model") == "| Model | claude-sonnet-5 | claude-sonnet-5 | — |" + assert "cross-model" not in md + + def test_claude_code_synthetic_model_is_not_a_model(self): + """A row that ended on a CLI-made error message records ``, not a model.""" + md = ExperimentReportGenerator.generate_experiment_report( + self._make_result({"a": ["", "claude-sonnet-5"], "b": ["claude-sonnet-5"], "c": [""]}) + ) + assert self._row(md, "Model") == "| Model | claude-sonnet-5 | claude-sonnet-5 | n/a |" + assert "cross-model" not in md + + def test_mixed_models_inside_one_variant_are_cross_model(self): + """A single variant whose rows ran two models pools them into one turn mean.""" + md = ExperimentReportGenerator.generate_experiment_report(self._make_result({"a": ["gpt-5", "claude-opus-5"]})) + assert self._row(md, "Model") == "| Model | claude-opus-5, gpt-5 |" + assert "| Assistant Turns (cross-model) |" in md + assert CROSS_MODEL_TURNS_NOTE in md + + def test_an_unrecorded_model_is_unknown_not_different(self): + md = ExperimentReportGenerator.generate_experiment_report( + self._make_result({"a": ["claude-sonnet-5"], "b": []}) + ) + assert self._row(md, "Model") == "| Model | claude-sonnet-5 | n/a | — |" + assert "cross-model" not in md + + def test_snapshot_id_is_the_same_model_as_its_alias(self): + """A crashed row keeps the configured alias; a replied row records the dated snapshot.""" + models = {"a": ["claude-haiku-4-5", "claude-haiku-4-5-20251001"], "b": ["claude-haiku-4-5-20251001"]} + md = ExperimentReportGenerator.generate_experiment_report(self._make_result(models)) + assert "cross-model" not in md + assert self._row(md, "Model") == "| Model | claude-haiku-4-5 | claude-haiku-4-5 | — |" + + def test_missing_models_used_key_loads_as_empty(self): + """An experiment.json written before models_used existed loads and renders as before.""" + result = self._make_result({"a": ["gpt-5"], "b": ["gpt-5"]}) + for ts in result.task_summaries: + ts.variant_results = [ + VariantResult.model_validate({k: v for k, v in vr.model_dump().items() if k != "models_used"}) + for vr in ts.variant_results + ] + assert all(vr.models_used == [] for vr in ts.variant_results) + assert "| Model |" not in ExperimentReportGenerator.generate_experiment_report(result) + + def test_no_recorded_model_renders_no_model_row(self): + md = ExperimentReportGenerator.generate_experiment_report(self._make_result({"a": [], "b": []})) + assert "| Model |" not in md + assert "cross-model" not in md + + def test_no_note_without_turn_data(self): + result = self._make_result({"a": ["gpt-5"], "b": ["claude-opus-5"]}) + for ts in result.task_summaries: + for vr in ts.variant_results: + vr.total_assistant_turns = None + md = ExperimentReportGenerator.generate_experiment_report(result) + assert "| Model | gpt-5 | claude-opus-5 | — |" in md + assert "Assistant Turns" not in md + assert CROSS_MODEL_TURNS_NOTE not in md + + def test_html_marks_the_same_rows_as_markdown(self): + from coder_eval.reports.html import _experiment_aggregate_metrics + + cross = _experiment_aggregate_metrics(self._make_result({"a": ["gpt-5"], "b": ["claude-opus-5"]})) + assert "Modelgpt-5claude-opus-5—" in cross + assert "Assistant Turns (cross-model)5.5 ± 0.725.5 ± 0.7—" in cross + assert html_escape(CROSS_MODEL_TURNS_NOTE) in cross + + same = _experiment_aggregate_metrics(self._make_result({"a": ["gpt-5"], "b": ["gpt-5"]})) + assert "Assistant Turns" in same + assert "cross-model" not in same + assert html_escape(CROSS_MODEL_TURNS_NOTE) not in same + + no_models = _experiment_aggregate_metrics(self._make_result({"a": [], "b": []})) + assert "Model" not in no_models + + no_turns = self._make_result({"a": ["gpt-5"], "b": ["claude-opus-5"]}) + for ts in no_turns.task_summaries: + for vr in ts.variant_results: + vr.total_assistant_turns = None + out = _experiment_aggregate_metrics(no_turns) + assert "Model" in out + assert "Assistant Turns" not in out + assert html_escape(CROSS_MODEL_TURNS_NOTE) not in out + + class TestPairedComparisonSection: """Tests for the ``## Paired Comparison`` markdown section.""" diff --git a/tests/test_experiment_runner.py b/tests/test_experiment_runner.py index 254055b7..bf127b8d 100644 --- a/tests/test_experiment_runner.py +++ b/tests/test_experiment_runner.py @@ -664,6 +664,23 @@ def test_three_replicates_collapse_to_single_variant_result(self): assert vr.replicate_count == 3 assert result.task_summaries[0].replicate_count == 3 + def test_models_used_lists_each_replicate_model_once(self): + """The fold keeps every distinct model, so a report can see a mixed-model variant.""" + reps = [self._make_tr("t", "v", score=1.0, replicate_index=r) for r in range(4)] + for rep, model in zip(reps, ["opus", None, "sonnet", "opus"], strict=True): + rep.result.model_used = model + result = aggregate_results( + experiment_id="e", description="", variant_ids=["v"], task_results=reps, total_duration=30.0 + ) + assert result.task_summaries[0].variant_results[0].models_used == ["opus", "sonnet"] + + def test_models_used_is_empty_when_no_replicate_recorded_one(self): + reps = [self._make_tr("t", "v", score=1.0, replicate_index=r) for r in range(2)] + result = aggregate_results( + experiment_id="e", description="", variant_ids=["v"], task_results=reps, total_duration=30.0 + ) + assert result.task_summaries[0].variant_results[0].models_used == [] + def test_mixed_statuses_picks_worst(self): reps = [ self._make_tr("t", "v", score=1.0, status="SUCCESS", replicate_index=0), diff --git a/tests/test_litellm_route.py b/tests/test_litellm_route.py index 3055ae44..0062ac12 100644 --- a/tests/test_litellm_route.py +++ b/tests/test_litellm_route.py @@ -26,7 +26,7 @@ ) from coder_eval.models.enums import ApiBackend from coder_eval.models.routing import ROUTE_NAMES, resolve_evaluation_route, resolve_route -from coder_eval.pricing import _normalize_model, calculate_cost +from coder_eval.pricing import calculate_cost, normalize_model def _make_agent(route, *, config_model: str | None = None) -> ClaudeCodeAgent: @@ -631,12 +631,12 @@ def test_kimi_converse_prefixed_prices_same(self): assert calculate_cost("converse/moonshotai.kimi-k2.5", 1_000_000, 1_000_000) == pytest.approx(0.72 + 3.6) def test_normalize_strips_converse_prefix(self): - assert _normalize_model("converse/zai.glm-5") == "zai.glm-5" - assert _normalize_model("bedrock/converse/deepseek.v3.2") == "deepseek.v3.2" + assert normalize_model("converse/zai.glm-5") == "zai.glm-5" + assert normalize_model("bedrock/converse/deepseek.v3.2") == "deepseek.v3.2" def test_normalize_identity_on_bare_ids(self): - assert _normalize_model("zai.glm-5") == "zai.glm-5" - assert _normalize_model("deepseek.v3.2") == "deepseek.v3.2" + assert normalize_model("zai.glm-5") == "zai.glm-5" + assert normalize_model("deepseek.v3.2") == "deepseek.v3.2" def test_converse_prefixed_id_prices_same(self): """The SDK reports model_used as e.g. 'converse/zai.glm-5' — must still price.""" diff --git a/tests/test_pricing_registry.py b/tests/test_pricing_registry.py index ecae0afc..a3c4d315 100644 --- a/tests/test_pricing_registry.py +++ b/tests/test_pricing_registry.py @@ -20,7 +20,7 @@ def test_registered_rate_resolves_through_calculate_cost(): def test_registered_bare_key_resolves_through_prefixed_lookup(): """register_pricing stores keys verbatim (bare), but calculate_cost resolves - via _normalize_model — so a bare-registered key must still price a + via normalize_model — so a bare-registered key must still price a Bedrock/vendor-prefixed model id. Guards against the two normalization seams (registration vs lookup) drifting apart.""" register_pricing({"acme-1": ModelPricing(1.0, 2.0, 3.0, 4.0)}) @@ -39,7 +39,7 @@ def test_registered_bare_key_resolves_through_prefixed_lookup(): ) def test_bedrock_version_suffix_prices_as_bare_key(model: str, bare: str): """Claude Code on Bedrock records the configured id, e.g. ``.env.example``'s ``…-v1:0``, as ``model_used``.""" - assert pricing._normalize_model(model) == bare + assert pricing.normalize_model(model) == bare assert calculate_cost(model, 1_000_000, 1_000_000) == calculate_cost(bare, 1_000_000, 1_000_000) > 0