Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .claude/notes/orchestration.md
Original file line number Diff line number Diff line change
Expand Up @@ -933,6 +933,18 @@ Ungraded replicates — and ONLY those — drop out of a mean entirely: `or 0.0`
a clean `execute` run down to a real-looking zero, while dropping an ERRORED one would pay
it a bonus.

The rule lives once, in `result_metrics.measured_scores`, read by `VariantAggregate.average_score`
and run.md's Avg Reliability Score. run.md also hides that score whenever `nothing_was_measured`
holds, as the pass rate beside it does. Two known gaps remain. Closing either one changes a
serialized field that downstream consumers read, so each is a separate change:

- `SuiteRollup.average_weighted_score` (suite.json) still averages scored rows only, so an
errored row leaves it.
- `aggregate_results` counts `VariantAggregate.tasks_measured` after `_fold_replicates` has
turned an errored replicate's missing score into 0.0, so an `execute` run with a crash
reports `pass_rate` 0.0 and `average_score` 0.0 in experiment.json where run.json reports
`None`.

Only graded variants can win or set a spread; including ungraded ones at 0.0 would name an
arbitrary "best" among scores that do not exist. When NOTHING was scored there is no
winner, and the fallback must not invent one — `variants[0]` is whichever arm the input
Expand Down
2 changes: 2 additions & 0 deletions docs/REPORT_SCHEMA.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ publishing different numbers for the same run.
| `tasks_cost_incomplete` | `int` | Rows whose recorded spend is missing money (unpriced model, or a hard kill that lost an in-flight turn). |
| `cost_complete` | `bool` | `tasks_cost_incomplete == 0`. When false, every cost figure above is a **floor**, not the bill. A run is never failed for this — see [Missing cost is never fatal](#missing-cost-is-never-fatal). |

run.md's **Avg Reliability Score** is not stored in `run.json`. It is the mean `weighted_score` over every row except `NOT_GRADED` ones, and a row with no score (`ERROR`, `BUILD_FAILED`, an unscored `TIMEOUT`) counts as `0.0`: the same rule as `VariantAggregate.average_score`. It is left out when no row produced a verdict, as `pass_rate` is.

### `task_results[]` — the flat per-task row

Each entry is an **untyped dict** (a denormalization, not a Pydantic model) with keys
Expand Down
12 changes: 12 additions & 0 deletions src/coder_eval/models/enums.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,18 @@ def category(self) -> Literal["succeeded", "failed", "error", "ungraded"]:
"""Classify this status into a reporting category (the SSOT for failed/succeeded/error)."""
return _STATUS_CATEGORIES[self]

@classmethod
def category_of(cls, status: str) -> Literal["succeeded", "failed", "error", "ungraded"]:
"""``category`` of a serialized status string, as read back from ``run.json``.

An unknown value (an older or newer schema) counts as ``error``: it is
classified explicitly rather than slipping past a denylist (CE018).
"""
try:
return cls(status).category
except ValueError:
return "error"

@property
def icon(self) -> str:
"""Single-character icon for reports and CLI output."""
Expand Down
24 changes: 4 additions & 20 deletions src/coder_eval/orchestration/experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
apply_prompt_mutations,
)
from ..path_utils import resolve_dir_template
from ..result_metrics import measured_scores
from .config import BatchRunConfig
from .config_merge import ConfigSource, Layer, merge_layers, resolve_root
from .task_loader import (
Expand Down Expand Up @@ -798,26 +799,9 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus:


def _measured_scores(rows: Sequence[VariantResult] | Sequence[TaskResult]) -> list[float]:
"""The scores of every row that was actually GRADED, errors included as 0.0.

The filter is on ``final_status.category``, deliberately not on
``weighted_score is not None``. Those look equivalent and are not: an
ERROR / BUILD_FAILED row also has no score, and dropping it removes it from
BOTH sides of the mean. A nightly where one image build failed would then
report a HIGHER headline score than a clean one, and A/B comparisons would
be biased toward whichever variant errored more — the exact "bonus for
erroring" the run-level rates are written to avoid.

Only ``ungraded`` rows leave both sides: nothing measured them, so they are
not a miss. Everything else that ran and produced no score IS a miss.
"""
scores: list[float] = []
for row in rows:
result = row if isinstance(row, VariantResult) else row.result
if result.final_status.category == "ungraded":
continue
scores.append(result.weighted_score if result.weighted_score is not None else 0.0)
return scores
"""``result_metrics.measured_scores`` over typed rows: errors count as 0.0, ungraded rows leave."""
results = [row if isinstance(row, VariantResult) else row.result for row in rows]
return measured_scores((r.final_status, r.weighted_score) for r in results)


def _mean_graded_score(vr_list: list[VariantResult]) -> float | None:
Expand Down
17 changes: 2 additions & 15 deletions src/coder_eval/reports/junit.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import re
import xml.etree.ElementTree as ET
from pathlib import Path, PurePosixPath, PureWindowsPath
from typing import Any, Literal
from typing import Any

from ..evaluation.judge_context import truncate
from ..models import FinalStatus, RunSummary, SuiteRollup
Expand All @@ -52,19 +52,6 @@ def _xml_safe(text: str) -> str:
return _ILLEGAL_XML.sub("", text)


def _category_of(status: str) -> Literal["succeeded", "failed", "error", "ungraded"]:
"""Map a serialized status string to a reporting category via the SSOT.

Goes through ``FinalStatus(value).category`` (an explicit allowlist, CE018);
an unknown status value (older/newer schema) falls to the error bucket
explicitly rather than via a denylist.
"""
try:
return FinalStatus(status).category
except ValueError:
return "error"


def _set_counts(elem: ET.Element, cases: list[ET.Element]) -> None:
"""Set ``tests``/``failures``/``errors``/``skipped`` from the actual children.

Expand Down Expand Up @@ -296,7 +283,7 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element:
ET.SubElement(properties, "property", {"name": key, "value": _xml_safe(str(value))})

status = _status_of(row)
category = _category_of(status)
category = FinalStatus.category_of(status)
if category == "succeeded":
return case

Expand Down
9 changes: 7 additions & 2 deletions src/coder_eval/reports/markdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
sum_costs,
)
from ..path_utils import TASK_JSON_FILENAME, build_task_run_dir
from ..result_metrics import measured_scores


if TYPE_CHECKING:
Expand Down Expand Up @@ -470,13 +471,14 @@ def _summary_section_lines(summary: RunSummary) -> list[str]:
]

# Aggregate P0 metrics
scores = [t["weighted_score"] for t in summary.task_results if t.get("weighted_score") is not None]
scores = measured_scores((t["status"], t.get("weighted_score")) for t in summary.task_results)
durations = [t["duration"] for t in summary.task_results if t["duration"] > 0]
similarities = [
t["reference_similarity"] for t in summary.task_results if t.get("reference_similarity") is not None
]

if scores:
# Gated like the pass rate: a 0.000 beside "Pass Rate: n/a" would read as "scored zero".
if scores and not nothing_was_measured(not_graded=summary.tasks_not_graded, measured=summary.tasks_measured):
lines.append(f"- **Avg Reliability Score**: {sum(scores) / len(scores):.3f}")
if durations:
# `duration` is END-TO-END (setup + agent + grading), so the label says so.
Expand Down Expand Up @@ -958,6 +960,9 @@ def _compute_suite_rollup(
# Verdict evidence, distinct from `rows_graded` (a bucket complement): a
# TIMEOUT counts as graded while no criterion ever ran on it.
rows_measured = len(scored)
# Not yet `measured_scores`: an errored row still leaves this mean. A known gap, kept apart
# because closing it changes suite.json's `average_weighted_score`.
# Rationale: .claude/notes/orchestration.md § Aggregation drops ungraded rows rather than zeroing them
average_weighted_score = sum(scored) / len(scored) if scored else None

# Per-criterion-type tallies (scores + errors) for the type-level summary.
Expand Down
19 changes: 18 additions & 1 deletion src/coder_eval/result_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from collections.abc import Iterable
from typing import NamedTuple

from coder_eval.models import AssistantMessage, EvaluationResult, TurnRecord
from coder_eval.models import AssistantMessage, EvaluationResult, FinalStatus, TurnRecord
from coder_eval.timing import main_thread_tool_spans, union_ms


Expand Down Expand Up @@ -117,6 +117,23 @@ def agent_wall_ms(result: EvaluationResult) -> float | None:
return sum(seconds) * 1000.0 if seconds else None


def measured_scores(rows: Iterable[tuple[str, float | None]]) -> list[float]:
"""The scores a headline mean is taken over, from ``(status, weighted_score)`` pairs.

Filters on the status category, not on ``weighted_score is not None``. An
``ungraded`` row was never measured, so it leaves both sides of the mean.
Any other row without a score (ERROR, BUILD_FAILED, an unscored TIMEOUT) was
attempted and missed, so it counts as ``0.0`` -- dropping it would let a run
report a higher mean for having errored. ``status`` may be a ``FinalStatus``
or a serialized string; an unknown string counts as ``error``.

Rationale: .claude/notes/orchestration.md § Aggregation drops ungraded rows rather than zeroing them
"""
return [
score if score is not None else 0.0 for status, score in rows if FinalStatus.category_of(status) != "ungraded"
]


def _sum_measured(values: Iterable[float | None]) -> float | None:
"""Sum what was measured, or ``None`` when nothing was.

Expand Down
58 changes: 57 additions & 1 deletion tests/test_reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import pytest

from coder_eval.models import RunSummary
from coder_eval.models import FinalStatus, RunSummary
from coder_eval.reports import ReportGenerator
from tests._fixtures.report_snapshots import assert_matches_snapshot

Expand Down Expand Up @@ -291,6 +291,8 @@ def test_generate_markdown_with_null_scores():
assert "task_error" in report_md
assert "ERROR" in report_md
assert "N/A" in report_md # Should show N/A for null weighted_score
# A graded run whose only row errored was attempted and missed: it scores 0.0.
assert "- **Avg Reliability Score**: 0.000" in report_md


def test_generate_markdown_no_similarity_column():
Expand Down Expand Up @@ -451,6 +453,60 @@ def test_generate_markdown_aggregate_metrics():
assert "Avg End-to-end Latency**: 50.0s" in report_md


def _score_line_summary(rows: list[tuple[str, float | None]]) -> RunSummary:
"""A RunSummary whose bucket counts and verdict evidence agree with ``rows``."""
categories = [FinalStatus.category_of(status) for status, _ in rows]
return RunSummary(
run_id="test-run",
start_time=datetime(2025, 10, 11, 12, 0, 0),
end_time=datetime(2025, 10, 11, 12, 2, 0),
total_duration_seconds=120.0,
tasks_run=len(rows),
tasks_succeeded=categories.count("succeeded"),
tasks_failed=categories.count("failed"),
tasks_error=categories.count("error"),
tasks_not_graded=categories.count("ungraded"),
tasks_measured=sum(1 for _, score in rows if score is not None),
task_results=[_make_task_result(f"task{i}", status, score, 40.0) for i, (status, score) in enumerate(rows)],
framework_version="0.1.0",
environment_info={},
)


_GRADED_ROWS: list[tuple[str, float | None]] = [("SUCCESS", 1.0), ("FAILURE", 0.0)]


@pytest.mark.parametrize(
("rows", "expected_score"),
[
pytest.param([*_GRADED_ROWS, ("ERROR", None)], "0.333", id="error-counts-as-zero"),
pytest.param([*_GRADED_ROWS, ("TIMEOUT", None)], "0.333", id="unscored-timeout-counts-as-zero"),
pytest.param([*_GRADED_ROWS, ("NO_SUCH_STATUS", None)], "0.333", id="unknown-status-counts-as-zero"),
pytest.param([*_GRADED_ROWS, ("NOT_GRADED", None)], "0.500", id="not-graded-leaves-the-mean"),
pytest.param([("ERROR", None), ("ERROR", None)], "0.000", id="all-error-run-scores-zero"),
pytest.param([("NOT_GRADED", None), ("NOT_GRADED", None)], None, id="all-not-graded-run-has-no-score"),
pytest.param([("NOT_GRADED", None)] * 9 + [("ERROR", None)], None, id="execute-run-with-a-crash-has-no-score"),
],
)
def test_avg_reliability_score_zeroes_unscored_rows_and_drops_ungraded(
rows: list[tuple[str, float | None]], expected_score: str | None
):
report_md = ReportGenerator.generate_markdown(_score_line_summary(rows))

if expected_score is None:
assert "Avg Reliability Score" not in report_md
else:
assert f"- **Avg Reliability Score**: {expected_score}" in report_md


def test_execute_run_with_a_crash_shows_no_score_beside_pass_rate_na():
"""A 0.000 next to "Pass Rate: n/a" would read as "measured and scored zero"."""
report_md = ReportGenerator.generate_markdown(_score_line_summary([("NOT_GRADED", None)] * 9 + [("ERROR", None)]))

assert "- **Pass Rate**: n/a" in report_md
assert "Avg Reliability Score" not in report_md


def test_generate_markdown_backward_compatible_task_results():
"""Test that old-format task_results (without new fields) still render correctly."""
summary = RunSummary(
Expand Down
41 changes: 41 additions & 0 deletions tests/test_ungraded_reporting.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,47 @@ def test_the_three_published_pass_rates_agree_on_one_set_of_counts() -> None:
assert aggregate.pass_rate is None


def test_run_md_and_the_variant_average_publish_one_mean_for_one_set_of_rows() -> None:
"""run.md showed 1.000 and experiment.md 0.500 for the same rows when each
surface kept its own copy of the averaging rule. Both now read
`measured_scores`; this pins that they agree on every status that matters."""
from coder_eval.reports import ReportGenerator

rows: list[tuple[FinalStatus, float | None]] = [
(FinalStatus.SUCCESS, 1.0),
(FinalStatus.FAILURE, 0.0),
(FinalStatus.ERROR, None),
(FinalStatus.TIMEOUT, None),
(FinalStatus.NOT_GRADED, None),
]
variant_rows = [
VariantResult(variant_id="v", task_id=f"t{i}", weighted_score=score, final_status=status, duration_seconds=1.0)
for i, (status, score) in enumerate(rows)
]
summary = _summary(
tasks_run=5,
tasks_succeeded=1,
tasks_failed=2,
tasks_error=1,
tasks_not_graded=1,
tasks_measured=2,
task_results=[
{"task_id": f"t{i}", "status": status, "weighted_score": score, "duration": 1.0}
for i, (status, score) in enumerate(rows)
],
)

variant_mean = _mean_graded_score(variant_rows)
assert variant_mean == 0.25
assert f"- **Avg Reliability Score**: {variant_mean:.3f}" in ReportGenerator.generate_markdown(summary)


def test_category_of_reads_a_serialized_status_and_counts_an_unknown_one_as_error() -> None:
assert FinalStatus.category_of("NOT_GRADED") == "ungraded"
assert FinalStatus.category_of("TIMEOUT") == "failed"
assert FinalStatus.category_of("NO_SUCH_STATUS") == "error"


def test_variant_aggregate_serializes_its_denominator() -> None:
"""A consumer that cannot read `tasks_graded` re-derives the rate and drifts."""
import json
Expand Down
Loading