Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/AB_EXPERIMENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,11 @@ The reports (generated by `reports/experiment.py`) are:
duration, total tokens, and — when `repeats > 1` — `per_replicate_scores` so you
can see variance, not just a point estimate). `experiment.html` renders the same
data for browsing.
The Aggregate Metrics table has a **Model** row: the models each variant ran on.
When the variants ran more than one model, the turns row reads **Assistant
Turns (cross-model)**, it has no p-value, and a note under the table says why:
a turn count depends on how much work a model batches into one call, so a turn
difference across models is mostly model style, not variant cost.
- **`<variant_id>/variant.md` / `variant.json`** — per-variant rollup (plus
`variant.html`).
- **`run.md` / `run.json`** (run root) — the flat batch log across all
Expand Down
5 changes: 4 additions & 1 deletion docs/REPORT_SCHEMA.md
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,10 @@ The cross-variant summary:
`{task_id, variant_results, best_variant, is_tie, score_spread, replicate_count}`,
where each `VariantResult` carries `{variant_id, task_id, weighted_score,
final_status, duration_seconds, total_tokens, iteration_count,
total_assistant_turns, reference_similarity, replicate_index, replicate_count}`.
total_assistant_turns, models_used, reference_similarity, replicate_index,
replicate_count}`. `models_used` is the sorted list of distinct `model_used`
values across the replicates (`[]` when none recorded a model, and in an
`experiment.json` written before the field existed).
`weighted_score` is `float | None`: `null` only when every replicate was
**ungraded**. An **errored** replicate counts as `0.0` — same rule as
`VariantAggregate.average_score` above. That is deliberate: dropping errored
Expand Down
8 changes: 8 additions & 0 deletions docs/TASK_DEFINITION_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,14 @@ The count compared against the budget is **visible turns** — one per tool call
plus one for the agent's final reply. It is *not* `total_turns`, which counts
model API calls (the `max_turns` unit), and one call can batch several tool calls.

Turn counts are comparable only **within one model**. Models differ in how much
work they batch into one call (one model chains a whole phase into a single
`Bash` call, another sends one probe per call), so the same task and prompt can
take twice the turns on a different model. Compare turns between runs or variants
on the same model. Cost and wall-clock time stay comparable across models. The
experiment report shows the model of each variant and marks a cross-model turn
comparison (see [Reading the Report](AB_EXPERIMENTS.md#reading-the-report)).

Set it to the number of turns a competent agent should need for the task. Pick
budgets consistently across a suite — the headline % is only comparable when
tasks are measured against realistic, like-for-like targets. Omit it (the
Expand Down
4 changes: 2 additions & 2 deletions evalboard/lib/pricing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ export interface Pricing {
export { PRICING };

// Strip the LiteLLM/Bedrock routing + region/vendor prefixes back to the bare
// pricing key — mirror of src/coder_eval/pricing.py::_normalize_model, since the
// pricing key — mirror of src/coder_eval/pricing.py::normalize_model, since the
// recorded model_used arrives qualified (e.g. "converse/zai.glm-5",
// "eu.anthropic.claude-sonnet-4-6"). Idempotent on already-bare ids.
// "openrouter/" is here for the same reason it is in _normalize_model: a harness
// "openrouter/" is here for the same reason it is in normalize_model: a harness
// that addresses OpenRouter natively (OpenCode) records the model WITH its
// provider prefix ("openrouter/deepseek/deepseek-v4-pro"), while the rate keys
// are the bare vendor/model ids the LiteLLM route records. Without the strip the
Expand Down
7 changes: 7 additions & 0 deletions src/coder_eval/models/experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,13 @@ class VariantResult(BaseModel): # noqa: CE009 -- persisted result model; round-
total_tokens: int | None = None
iteration_count: int | None = None
total_assistant_turns: int | None = None
models_used: list[str] = Field(
default_factory=list,
description=(
"Distinct `model_used` values across the folded replicates, sorted. Empty when no replicate "
"recorded a model. The experiment report reads it to mark a cross-model turn comparison."
),
)
reference_similarity: float | None = None
replicate_index: int = Field(
default=0,
Expand Down
1 change: 1 addition & 0 deletions src/coder_eval/orchestration/experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -845,6 +845,7 @@ def _fold_replicates(task_id: str, variant_id: str, reps: list[TaskResult]) -> V
total_tokens=sum(token_vals) if token_vals else None,
iteration_count=round(sum(iter_counts) / len(iter_counts)) if iter_counts else None,
total_assistant_turns=round(sum(asst_turns) / len(asst_turns)) if asst_turns else None,
models_used=sorted({r.result.model_used for r in reps if r.result.model_used}),
reference_similarity=_mean_reference_similarity(reps),
replicate_index=0, # aggregate — points at first replicate for link rendering
replicate_count=len(reps),
Expand Down
6 changes: 3 additions & 3 deletions src/coder_eval/pricing.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def register_pricing(rates: dict[str, ModelPricing]) -> None:
_BEDROCK_VERSION_SUFFIX = re.compile(r"-v\d+:\d+$")


def _normalize_model(model: str) -> str:
def normalize_model(model: str) -> str:
"""Strip Bedrock region + vendor qualifiers and version suffix back to the bare pricing key.

``eu.anthropic.claude-opus-4-8`` → ``claude-opus-4-8``;
Expand Down Expand Up @@ -245,7 +245,7 @@ def _normalize_model(model: str) -> str:

def is_priced(model: str) -> bool:
"""Whether the rate card can price this model (after prefix normalization)."""
return _lookup_rate(_normalize_model(model)) is not None
return _lookup_rate(normalize_model(model)) is not None


def unpriced_models(models: Iterable[str | None]) -> list[str]:
Expand Down Expand Up @@ -275,7 +275,7 @@ def calculate_cost(
the same as the bare ``claude-opus-4-8``. Returns None if the (normalized)
model is not in the pricing table.
"""
pricing = _lookup_rate(_normalize_model(model))
pricing = _lookup_rate(normalize_model(model))
if pricing is None:
return None

Expand Down
32 changes: 28 additions & 4 deletions src/coder_eval/reports/experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from ..run_record import eval_result_to_task_dict
from ..stats import bootstrap_mean_ci, stddev, welch_t_test, wilson_interval
from .helpers import (
CROSS_MODEL_TURNS_NOTE,
VariantSeries,
collect_variant_series,
describe_prompt_config,
Expand All @@ -23,6 +24,8 @@
is_env_table_key,
load_variant_eval_results,
paired_comparison,
turns_cross_model,
variant_models,
)
from .html import write_experiment_html, write_variant_html
from .markdown import ReportGenerator, resolve_agent_settings, resolve_context_window
Expand Down Expand Up @@ -189,10 +192,13 @@ def _aggregate_stat_rows(
show_p_values: bool,
vid_a: str,
vid_b: str,
*,
cross_model: bool,
) -> list[str]:
"""The mean ± stddev rows of the Aggregate Metrics table (Score, Avg Duration,
and the optional Assistant Turns / Tokens rows), each with a Welch t-test
p-value when ``show_p_values``."""
p-value when ``show_p_values``. When ``cross_model``, the Assistant Turns row
is marked and has no p-value."""
lines: list[str] = []

# Row: Score (mean ± stddev, p-value)
Expand All @@ -215,11 +221,11 @@ def _aggregate_stat_rows(

# Row: Assistant Turns (if data available)
if any(series[vid].asst_turns for vid in result.variant_ids):
row = "| Assistant Turns"
row = "| Assistant Turns (cross-model)" if cross_model else "| Assistant Turns"
for vid in result.variant_ids:
row += f" | {fmt_mean_sd(series[vid].asst_turns, '.1f')}"
if show_p_values:
p = welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)
p = None if cross_model else welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)
row += f" | {fmt_p(p)}"
lines.append(row + " |")

Expand Down Expand Up @@ -256,8 +262,23 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]:
sep += "|"

lines = ["", "## Aggregate Metrics", "", header, sep]

# Row: Model (if any variant recorded one). Turns depend on the model, so
# the table that shows them also shows what each variant ran on.
models = variant_models(result)
cross_model = turns_cross_model(models)
if any(models.values()):
row = "| Model"
for vid in result.variant_ids:
row += f" | {', '.join(models[vid]) or 'n/a'}"
if show_p_values:
row += " | —"
lines.append(row + " |")

lines += ExperimentReportGenerator._aggregate_count_rows(result, show_p_values)
lines += ExperimentReportGenerator._aggregate_stat_rows(result, series, show_p_values, vid_a, vid_b)
lines += ExperimentReportGenerator._aggregate_stat_rows(
result, series, show_p_values, vid_a, vid_b, cross_model=cross_model
)

# Row: Replicates/task (if any variant ran >1 replicate)
if any(result.variant_aggregates[vid].replicate_count > 1 for vid in result.variant_ids):
Expand All @@ -269,6 +290,9 @@ def _aggregate_metrics_lines(result: ExperimentResult) -> list[str]:
row += " | —"
lines.append(row + " |")

if cross_model and any(series[vid].asst_turns for vid in result.variant_ids):
lines += ["", f"*{CROSS_MODEL_TURNS_NOTE}*"]

return lines

@staticmethod
Expand Down
50 changes: 50 additions & 0 deletions src/coder_eval/reports/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from __future__ import annotations

import logging
import re
from pathlib import Path
from typing import NamedTuple

Expand All @@ -26,6 +27,7 @@
TaskExperimentSummary,
)
from ..path_utils import TASK_JSON_FILENAME
from ..pricing import normalize_model
from ..stats import cohens_d, mean, paired_t_ci, paired_t_test, stddev


Expand Down Expand Up @@ -126,6 +128,54 @@ def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries]
return series


# Shown under an Aggregate Metrics table whose Assistant Turns row compares more
# than one model. Shared so the markdown and HTML reports say the same thing.
CROSS_MODEL_TURNS_NOTE = (
"These turn counts come from more than one model. How many turns a task takes depends on how much"
+ " work a model batches into one call, so a turn difference across models is mostly model style,"
+ " not variant cost. Compare turns only between variants on the same model."
)


# A dated snapshot id (`claude-haiku-4-5-20251001`) is the same model as its alias
# (`claude-haiku-4-5`). A row that replied records the resolved snapshot, a row that
# crashed first keeps the configured alias.
_SNAPSHOT_DATE_SUFFIX = re.compile(r"-20\d{6}$")


# Claude Code stamps this on an assistant message it makes up itself (an API
# error it reports without a model call), and a row that ends on one records it
# as `model_used`. It names no model, so it never counts as one.
_SYNTHETIC_MODEL = "<synthetic>"


def variant_models(result: ExperimentResult) -> dict[str, list[str]]:
"""The distinct models each variant ran on, sorted, keyed by variant id.

Ids are normalized with ``pricing.normalize_model`` and a trailing snapshot date
is dropped: a row that errored before the model resolved keeps the qualified route
id (``eu.anthropic.claude-sonnet-5``) or the configured alias, and a raw comparison
would read either as a second model.
"""
models: dict[str, set[str]] = {vid: set() for vid in result.variant_ids}
for ts in result.task_summaries:
for vr in ts.variant_results:
if vr.variant_id in models:
models[vr.variant_id].update(
_SNAPSHOT_DATE_SUFFIX.sub("", normalize_model(m)) for m in vr.models_used if m != _SYNTHETIC_MODEL
)
return {vid: sorted(found) for vid, found in models.items()}


def turns_cross_model(models: dict[str, list[str]]) -> bool:
"""Whether the variants' turn counts come from more than one model.

A variant with no recorded model is unknown, not different, so it never
makes the comparison cross-model on its own.
"""
return len({m for found in models.values() for m in found}) > 1


class PairedComparison(NamedTuple):
"""A 2-variant paired comparison over per-task mean scores.

Expand Down
21 changes: 17 additions & 4 deletions src/coder_eval/reports/html.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from ..result_metrics import agent_wall_ms, expected_turns_overage, turn_time_buckets
from ..stats import stddev, welch_t_test
from .helpers import (
CROSS_MODEL_TURNS_NOTE,
collect_variant_series,
describe_prompt_config,
fmt_mean_sd,
Expand All @@ -35,6 +36,8 @@
is_env_table_key,
load_variant_eval_results,
paired_comparison,
turns_cross_model,
variant_models,
)
from .markdown import (
SLOW_PARAMS_PREVIEW_CHARS,
Expand Down Expand Up @@ -1386,6 +1389,12 @@ def _row(label: str, values: list[str], p: str | None) -> str:
return "<tr>" + "".join(cells) + "</tr>"

rows: list[str] = []
# Turns depend on the model, so the table that shows them also shows what
# each variant ran on.
models = variant_models(result)
cross_model = turns_cross_model(models)
if any(models.values()):
rows.append(_row("Model", [", ".join(models[vid]) or "n/a" for vid in result.variant_ids], None))
rows.append(_row("Tasks Run", [str(result.variant_aggregates[vid].tasks_run) for vid in result.variant_ids], None))
rows.append(
_row(
Expand Down Expand Up @@ -1427,12 +1436,15 @@ def _pass_rate(vid: str) -> str:
fmt_p(welch_t_test(series[vid_a].durations, series[vid_b].durations)) if show_p else None,
)
)
if any(series[vid].asst_turns for vid in result.variant_ids):
has_turns = any(series[vid].asst_turns for vid in result.variant_ids)
if has_turns:
rows.append(
_row(
"Assistant Turns",
"Assistant Turns (cross-model)" if cross_model else "Assistant Turns",
[fmt_mean_sd(series[vid].asst_turns, ".1f") for vid in result.variant_ids],
fmt_p(welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns)) if show_p else None,
fmt_p(welch_t_test(series[vid_a].asst_turns, series[vid_b].asst_turns))
if show_p and not cross_model
else None,
)
)
if any(series[vid].tokens for vid in result.variant_ids):
Expand All @@ -1444,14 +1456,15 @@ def _pass_rate(vid: str) -> str:
)
)

note = f'\n<p class="muted">{_esc(CROSS_MODEL_TURNS_NOTE)}</p>' if cross_model and has_turns else ""
return f"""
<h2>Aggregate Metrics</h2>
<div class="card" style="padding:0">
<table>
<thead>{header_html}</thead>
<tbody>{"".join(rows)}</tbody>
</table>
</div>
</div>{note}
"""


Expand Down
Loading