diff --git a/.claude/notes/orchestration.md b/.claude/notes/orchestration.md index 893c3577..074e4be3 100644 --- a/.claude/notes/orchestration.md +++ b/.claude/notes/orchestration.md @@ -933,6 +933,18 @@ Ungraded replicates — and ONLY those — drop out of a mean entirely: `or 0.0` a clean `execute` run down to a real-looking zero, while dropping an ERRORED one would pay it a bonus. +The rule lives once, in `result_metrics.measured_scores`, read by `VariantAggregate.average_score` +and run.md's Avg Reliability Score. run.md also hides that score whenever `nothing_was_measured` +holds, as the pass rate beside it does. Two known gaps remain. Closing either one changes a +serialized field that downstream consumers read, so each is a separate change: + +- `SuiteRollup.average_weighted_score` (suite.json) still averages scored rows only, so an + errored row leaves it. +- `aggregate_results` counts `VariantAggregate.tasks_measured` after `_fold_replicates` has + turned an errored replicate's missing score into 0.0, so an `execute` run with a crash + reports `pass_rate` 0.0 and `average_score` 0.0 in experiment.json where run.json reports + `None`. + Only graded variants can win or set a spread; including ungraded ones at 0.0 would name an arbitrary "best" among scores that do not exist. When NOTHING was scored there is no winner, and the fallback must not invent one — `variants[0]` is whichever arm the input diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index a6e96bab..c1e11220 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -71,6 +71,8 @@ publishing different numbers for the same run. | `tasks_cost_incomplete` | `int` | Rows whose recorded spend is missing money (unpriced model, or a hard kill that lost an in-flight turn). | | `cost_complete` | `bool` | `tasks_cost_incomplete == 0`. When false, every cost figure above is a **floor**, not the bill. A run is never failed for this — see [Missing cost is never fatal](#missing-cost-is-never-fatal). | +run.md's **Avg Reliability Score** is not stored in `run.json`. It is the mean `weighted_score` over every row except `NOT_GRADED` ones, and a row with no score (`ERROR`, `BUILD_FAILED`, an unscored `TIMEOUT`) counts as `0.0`: the same rule as `VariantAggregate.average_score`. It is left out when no row produced a verdict, as `pass_rate` is. + ### `task_results[]` — the flat per-task row Each entry is an **untyped dict** (a denormalization, not a Pydantic model) with keys diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index eeaca391..f2d22fac 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -25,6 +25,18 @@ def category(self) -> Literal["succeeded", "failed", "error", "ungraded"]: """Classify this status into a reporting category (the SSOT for failed/succeeded/error).""" return _STATUS_CATEGORIES[self] + @classmethod + def category_of(cls, status: str) -> Literal["succeeded", "failed", "error", "ungraded"]: + """``category`` of a serialized status string, as read back from ``run.json``. + + An unknown value (an older or newer schema) counts as ``error``: it is + classified explicitly rather than slipping past a denylist (CE018). + """ + try: + return cls(status).category + except ValueError: + return "error" + @property def icon(self) -> str: """Single-character icon for reports and CLI output.""" diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 2b24266b..f93b84d2 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -39,6 +39,7 @@ apply_prompt_mutations, ) from ..path_utils import resolve_dir_template +from ..result_metrics import measured_scores from .config import BatchRunConfig from .config_merge import ConfigSource, Layer, merge_layers, resolve_root from .task_loader import ( @@ -798,26 +799,9 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: def _measured_scores(rows: Sequence[VariantResult] | Sequence[TaskResult]) -> list[float]: - """The scores of every row that was actually GRADED, errors included as 0.0. - - The filter is on ``final_status.category``, deliberately not on - ``weighted_score is not None``. Those look equivalent and are not: an - ERROR / BUILD_FAILED row also has no score, and dropping it removes it from - BOTH sides of the mean. A nightly where one image build failed would then - report a HIGHER headline score than a clean one, and A/B comparisons would - be biased toward whichever variant errored more — the exact "bonus for - erroring" the run-level rates are written to avoid. - - Only ``ungraded`` rows leave both sides: nothing measured them, so they are - not a miss. Everything else that ran and produced no score IS a miss. - """ - scores: list[float] = [] - for row in rows: - result = row if isinstance(row, VariantResult) else row.result - if result.final_status.category == "ungraded": - continue - scores.append(result.weighted_score if result.weighted_score is not None else 0.0) - return scores + """``result_metrics.measured_scores`` over typed rows: errors count as 0.0, ungraded rows leave.""" + results = [row if isinstance(row, VariantResult) else row.result for row in rows] + return measured_scores((r.final_status, r.weighted_score) for r in results) def _mean_graded_score(vr_list: list[VariantResult]) -> float | None: diff --git a/src/coder_eval/reports/junit.py b/src/coder_eval/reports/junit.py index 901a9712..cc08ce7c 100644 --- a/src/coder_eval/reports/junit.py +++ b/src/coder_eval/reports/junit.py @@ -25,7 +25,7 @@ import re import xml.etree.ElementTree as ET from pathlib import Path, PurePosixPath, PureWindowsPath -from typing import Any, Literal +from typing import Any from ..evaluation.judge_context import truncate from ..models import FinalStatus, RunSummary, SuiteRollup @@ -52,19 +52,6 @@ def _xml_safe(text: str) -> str: return _ILLEGAL_XML.sub("", text) -def _category_of(status: str) -> Literal["succeeded", "failed", "error", "ungraded"]: - """Map a serialized status string to a reporting category via the SSOT. - - Goes through ``FinalStatus(value).category`` (an explicit allowlist, CE018); - an unknown status value (older/newer schema) falls to the error bucket - explicitly rather than via a denylist. - """ - try: - return FinalStatus(status).category - except ValueError: - return "error" - - def _set_counts(elem: ET.Element, cases: list[ET.Element]) -> None: """Set ``tests``/``failures``/``errors``/``skipped`` from the actual children. @@ -296,7 +283,7 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: ET.SubElement(properties, "property", {"name": key, "value": _xml_safe(str(value))}) status = _status_of(row) - category = _category_of(status) + category = FinalStatus.category_of(status) if category == "succeeded": return case diff --git a/src/coder_eval/reports/markdown.py b/src/coder_eval/reports/markdown.py index 6146729c..c2d6fbfc 100644 --- a/src/coder_eval/reports/markdown.py +++ b/src/coder_eval/reports/markdown.py @@ -27,6 +27,7 @@ sum_costs, ) from ..path_utils import TASK_JSON_FILENAME, build_task_run_dir +from ..result_metrics import measured_scores if TYPE_CHECKING: @@ -470,13 +471,14 @@ def _summary_section_lines(summary: RunSummary) -> list[str]: ] # Aggregate P0 metrics - scores = [t["weighted_score"] for t in summary.task_results if t.get("weighted_score") is not None] + scores = measured_scores((t["status"], t.get("weighted_score")) for t in summary.task_results) durations = [t["duration"] for t in summary.task_results if t["duration"] > 0] similarities = [ t["reference_similarity"] for t in summary.task_results if t.get("reference_similarity") is not None ] - if scores: + # Gated like the pass rate: a 0.000 beside "Pass Rate: n/a" would read as "scored zero". + if scores and not nothing_was_measured(not_graded=summary.tasks_not_graded, measured=summary.tasks_measured): lines.append(f"- **Avg Reliability Score**: {sum(scores) / len(scores):.3f}") if durations: # `duration` is END-TO-END (setup + agent + grading), so the label says so. @@ -958,6 +960,9 @@ def _compute_suite_rollup( # Verdict evidence, distinct from `rows_graded` (a bucket complement): a # TIMEOUT counts as graded while no criterion ever ran on it. rows_measured = len(scored) + # Not yet `measured_scores`: an errored row still leaves this mean. A known gap, kept apart + # because closing it changes suite.json's `average_weighted_score`. + # Rationale: .claude/notes/orchestration.md § Aggregation drops ungraded rows rather than zeroing them average_weighted_score = sum(scored) / len(scored) if scored else None # Per-criterion-type tallies (scores + errors) for the type-level summary. diff --git a/src/coder_eval/result_metrics.py b/src/coder_eval/result_metrics.py index 3dde1b30..93609be0 100644 --- a/src/coder_eval/result_metrics.py +++ b/src/coder_eval/result_metrics.py @@ -17,7 +17,7 @@ from collections.abc import Iterable from typing import NamedTuple -from coder_eval.models import AssistantMessage, EvaluationResult, TurnRecord +from coder_eval.models import AssistantMessage, EvaluationResult, FinalStatus, TurnRecord from coder_eval.timing import main_thread_tool_spans, union_ms @@ -117,6 +117,23 @@ def agent_wall_ms(result: EvaluationResult) -> float | None: return sum(seconds) * 1000.0 if seconds else None +def measured_scores(rows: Iterable[tuple[str, float | None]]) -> list[float]: + """The scores a headline mean is taken over, from ``(status, weighted_score)`` pairs. + + Filters on the status category, not on ``weighted_score is not None``. An + ``ungraded`` row was never measured, so it leaves both sides of the mean. + Any other row without a score (ERROR, BUILD_FAILED, an unscored TIMEOUT) was + attempted and missed, so it counts as ``0.0`` -- dropping it would let a run + report a higher mean for having errored. ``status`` may be a ``FinalStatus`` + or a serialized string; an unknown string counts as ``error``. + + Rationale: .claude/notes/orchestration.md § Aggregation drops ungraded rows rather than zeroing them + """ + return [ + score if score is not None else 0.0 for status, score in rows if FinalStatus.category_of(status) != "ungraded" + ] + + def _sum_measured(values: Iterable[float | None]) -> float | None: """Sum what was measured, or ``None`` when nothing was. diff --git a/tests/test_reports.py b/tests/test_reports.py index 4989f92c..f9460a3f 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -5,7 +5,7 @@ import pytest -from coder_eval.models import RunSummary +from coder_eval.models import FinalStatus, RunSummary from coder_eval.reports import ReportGenerator from tests._fixtures.report_snapshots import assert_matches_snapshot @@ -291,6 +291,8 @@ def test_generate_markdown_with_null_scores(): assert "task_error" in report_md assert "ERROR" in report_md assert "N/A" in report_md # Should show N/A for null weighted_score + # A graded run whose only row errored was attempted and missed: it scores 0.0. + assert "- **Avg Reliability Score**: 0.000" in report_md def test_generate_markdown_no_similarity_column(): @@ -451,6 +453,60 @@ def test_generate_markdown_aggregate_metrics(): assert "Avg End-to-end Latency**: 50.0s" in report_md +def _score_line_summary(rows: list[tuple[str, float | None]]) -> RunSummary: + """A RunSummary whose bucket counts and verdict evidence agree with ``rows``.""" + categories = [FinalStatus.category_of(status) for status, _ in rows] + return RunSummary( + run_id="test-run", + start_time=datetime(2025, 10, 11, 12, 0, 0), + end_time=datetime(2025, 10, 11, 12, 2, 0), + total_duration_seconds=120.0, + tasks_run=len(rows), + tasks_succeeded=categories.count("succeeded"), + tasks_failed=categories.count("failed"), + tasks_error=categories.count("error"), + tasks_not_graded=categories.count("ungraded"), + tasks_measured=sum(1 for _, score in rows if score is not None), + task_results=[_make_task_result(f"task{i}", status, score, 40.0) for i, (status, score) in enumerate(rows)], + framework_version="0.1.0", + environment_info={}, + ) + + +_GRADED_ROWS: list[tuple[str, float | None]] = [("SUCCESS", 1.0), ("FAILURE", 0.0)] + + +@pytest.mark.parametrize( + ("rows", "expected_score"), + [ + pytest.param([*_GRADED_ROWS, ("ERROR", None)], "0.333", id="error-counts-as-zero"), + pytest.param([*_GRADED_ROWS, ("TIMEOUT", None)], "0.333", id="unscored-timeout-counts-as-zero"), + pytest.param([*_GRADED_ROWS, ("NO_SUCH_STATUS", None)], "0.333", id="unknown-status-counts-as-zero"), + pytest.param([*_GRADED_ROWS, ("NOT_GRADED", None)], "0.500", id="not-graded-leaves-the-mean"), + pytest.param([("ERROR", None), ("ERROR", None)], "0.000", id="all-error-run-scores-zero"), + pytest.param([("NOT_GRADED", None), ("NOT_GRADED", None)], None, id="all-not-graded-run-has-no-score"), + pytest.param([("NOT_GRADED", None)] * 9 + [("ERROR", None)], None, id="execute-run-with-a-crash-has-no-score"), + ], +) +def test_avg_reliability_score_zeroes_unscored_rows_and_drops_ungraded( + rows: list[tuple[str, float | None]], expected_score: str | None +): + report_md = ReportGenerator.generate_markdown(_score_line_summary(rows)) + + if expected_score is None: + assert "Avg Reliability Score" not in report_md + else: + assert f"- **Avg Reliability Score**: {expected_score}" in report_md + + +def test_execute_run_with_a_crash_shows_no_score_beside_pass_rate_na(): + """A 0.000 next to "Pass Rate: n/a" would read as "measured and scored zero".""" + report_md = ReportGenerator.generate_markdown(_score_line_summary([("NOT_GRADED", None)] * 9 + [("ERROR", None)])) + + assert "- **Pass Rate**: n/a" in report_md + assert "Avg Reliability Score" not in report_md + + def test_generate_markdown_backward_compatible_task_results(): """Test that old-format task_results (without new fields) still render correctly.""" summary = RunSummary( diff --git a/tests/test_ungraded_reporting.py b/tests/test_ungraded_reporting.py index c232e558..315785c5 100644 --- a/tests/test_ungraded_reporting.py +++ b/tests/test_ungraded_reporting.py @@ -112,6 +112,47 @@ def test_the_three_published_pass_rates_agree_on_one_set_of_counts() -> None: assert aggregate.pass_rate is None +def test_run_md_and_the_variant_average_publish_one_mean_for_one_set_of_rows() -> None: + """run.md showed 1.000 and experiment.md 0.500 for the same rows when each + surface kept its own copy of the averaging rule. Both now read + `measured_scores`; this pins that they agree on every status that matters.""" + from coder_eval.reports import ReportGenerator + + rows: list[tuple[FinalStatus, float | None]] = [ + (FinalStatus.SUCCESS, 1.0), + (FinalStatus.FAILURE, 0.0), + (FinalStatus.ERROR, None), + (FinalStatus.TIMEOUT, None), + (FinalStatus.NOT_GRADED, None), + ] + variant_rows = [ + VariantResult(variant_id="v", task_id=f"t{i}", weighted_score=score, final_status=status, duration_seconds=1.0) + for i, (status, score) in enumerate(rows) + ] + summary = _summary( + tasks_run=5, + tasks_succeeded=1, + tasks_failed=2, + tasks_error=1, + tasks_not_graded=1, + tasks_measured=2, + task_results=[ + {"task_id": f"t{i}", "status": status, "weighted_score": score, "duration": 1.0} + for i, (status, score) in enumerate(rows) + ], + ) + + variant_mean = _mean_graded_score(variant_rows) + assert variant_mean == 0.25 + assert f"- **Avg Reliability Score**: {variant_mean:.3f}" in ReportGenerator.generate_markdown(summary) + + +def test_category_of_reads_a_serialized_status_and_counts_an_unknown_one_as_error() -> None: + assert FinalStatus.category_of("NOT_GRADED") == "ungraded" + assert FinalStatus.category_of("TIMEOUT") == "failed" + assert FinalStatus.category_of("NO_SUCH_STATUS") == "error" + + def test_variant_aggregate_serializes_its_denominator() -> None: """A consumer that cannot read `tasks_graded` re-derives the rate and drifts.""" import json