From 076628ba224fcdc27da04980882af6717ef4d447 Mon Sep 17 00:00:00 2001 From: NopeCharon <115794254+NopeCharon@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:58:18 +0530 Subject: [PATCH] feat(criteria): add report_only to exclude a weighted criterion from scoring A criterion could only be made informational by setting weight: 0, which rewrites the declared weight that some evaluation contracts must keep for denominator or provenance parity. pass_threshold: 0.0 was no substitute: it keeps the criterion out of final_status, but its score and weight still count in weighted_score, and the result is persisted as gating. Add report_only (default false) to every criterion. A report_only criterion still runs and keeps its raw score, weight and pass_threshold in the record, but BaseSuccessCriterion.is_gating is now false for it, so the existing single source of truth carries it everywhere: - calculate_weighted_score skips non-gating criteria, so the score covers exactly the criteria the gate reads (weight: 0 already contributed nothing, so its results are unchanged). - all_criteria_passed, CriterionResult.gating, the evaluate summary, the HTML and JUnit reports, the suite rollup and the evalboard INFO pill all follow is_gating already. - Like weight: 0, report_only may not be combined with a stop_early block or suite_thresholds; the validator now names whichever cause applies. The surface labels that said "weight: 0" now say "informational", since a non-gating result no longer implies a zero weight. Docs: the Task Definition Guide documents the field with an example and how it differs from pass_threshold: 0.0; the report schema, the extending guide, the task and lint-tasks skills follow, and the plugin criteria reference is regenerated. The Harbor export golden task.yaml gains the new default field. Closes #101 --- docs/EXTENDING.md | 1 + docs/REPORT_SCHEMA.md | 6 +- docs/TASK_DEFINITION_GUIDE.md | 35 +++++++-- evalboard/app/runs/[id]/[...task]/_chips.tsx | 5 +- evalboard/lib/runs.ts | 6 +- plugins/coder-eval/reference/criteria.md | 3 +- plugins/coder-eval/skills/lint-tasks/SKILL.md | 14 ++-- plugins/coder-eval/skills/task/SKILL.md | 2 +- src/coder_eval/cli/evaluate_command.py | 6 +- src/coder_eval/evaluation/checker.py | 2 +- src/coder_eval/models/criteria.py | 78 ++++++++++++------- src/coder_eval/models/results.py | 19 +++-- src/coder_eval/reports/html.py | 4 +- src/coder_eval/reports/markdown.py | 2 +- .../expected/tests/task.yaml | 2 + .../test_task_report_only_criterion.yaml | 20 +++++ tests/test_evaluate_command.py | 29 +++++++ tests/test_threshold_enforcement.py | 65 +++++++++++++++- 18 files changed, 232 insertions(+), 67 deletions(-) create mode 100644 tests/fixtures/tasks/test_task_report_only_criterion.yaml diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index bef3866db..0584b7ac5 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -170,6 +170,7 @@ class MyCriterion(BaseSuccessCriterion): Union membership is required — a run validates that every union member's `type` has a registered checker, and rejects unknown `type` tags in YAML. `BaseSuccessCriterion` gives you `description`, `weight` (default 1.0; `0` = informational/non-gating), +`report_only` (default `false`; `true` = informational at the declared weight), `pass_threshold` (default 0.9) and `suite_thresholds` for free, with `extra="forbid"` so YAML typos are caught. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index c1e112206..88af88a5d 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -211,9 +211,9 @@ A discriminated union on `result_kind` (`basic` / `judge` / `classification`). B `criterion_type`, `description`, `score` (0.0–1.0), `details`, `error`, `evaluation_status` (`evaluated` by default; `not_evaluated` means the check did not run and is distinct from an evaluated 0.0), -`pass_threshold` (default 0.9), `gating` (default `true`; `false` = informational / -weight-0, excluded from the score and the pass/fail gate). The base allows extra -fields so subclass keys round-trip. +`pass_threshold` (default 0.9), `gating` (default `true`; `false` = informational, from +`weight: 0` or `report_only: true`, excluded from the score and the pass/fail gate). The +base allows extra fields so subclass keys round-trip. - **`classification`** adds `observed_label`, `expected_label` (sentinels like `(none)` / `(other)` allowed). Emitted by `classification_match`, `skill_triggered`. diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 8c9481567..363c9c4fa 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -749,6 +749,7 @@ All criteria share these fields: |-------|---------|-------------| | `description` | — | Human-readable description (required) | | `weight` | 1.0 | Relative importance for weighted score. `0` = **informational**: excluded from both the score and the pass/fail gate | +| `report_only` | `false` | `true` = **informational** without changing the declared `weight`: excluded from both the score and the pass/fail gate, like `weight: 0` | | `pass_threshold` | 0.9 | Minimum score (0.0–1.0) to pass | | `stop_early` | `null` | **Only on live-observable criteria** (`skill_triggered`, `command_executed`). Presence arms the criterion for early stop (no run-level switch needed): an effective fail may end the run (weighted ceiling rule, recall deferral). Keys: `on_pass: stop\|continue` (default `continue`), `decide_within: N` (timeout → effective fail, reported as `decision_budget_exceeded`). Inert triggers by design on instances that can't decide their polarity (dataset fan-out support). See [`stop_early`](#stop_early-opt-in-early-stop). | @@ -758,11 +759,33 @@ All criteria share these fields: - **Continuous** (0.0–1.0): `reference_comparison`, `commands_efficiency`, `llm_judge`, `agent_judge`, `system_one_judge` **Task success:** all *gating* criteria must score >= their `pass_threshold`. A -criterion with `weight: 0` is informational — it is still checked, stored, and -rendered in reports, but it neither contributes to the score nor fails the task. -(A `weight: 0` criterion may not set a `stop_early` block or `suite_thresholds`: arming a -non-gating criterion for the early-stop or suite gate would let an -"informational" check flip a run to failure.) +criterion with `weight: 0` or `report_only: true` is informational — it is still +checked, stored, and rendered in reports, but it neither contributes to the score nor +fails the task. (An informational criterion may not set a `stop_early` block or +`suite_thresholds`: arming a non-gating criterion for the early-stop or suite gate would +let an "informational" check flip a run to failure.) + +Use `report_only: true` when the criterion must keep its declared `weight` — for example, +to match the weights of another evaluation contract. A `pass_threshold: 0.0` is not the +same thing: that criterion always passes the gate, but its score and weight still count +in `weighted_score`. + +```yaml +success_criteria: + - type: run_command + description: Unit tests pass + command: pytest -q + weight: 8 + pass_threshold: 1.0 + - type: run_command + description: Linter is clean (recorded, not scored) + command: ruff check . + weight: 1 + report_only: true +``` + +When the tests pass, a failed `ruff check` leaves the task at `SUCCESS` with +`weighted_score: 1.0` (not 8/9), and its result is stored with `gating: false`. Every surface labels it as such rather than as a failure: the terminal shows `○` instead of `✓`/`✗`, the HTML report tags the row "informational — not gated" and @@ -772,7 +795,7 @@ suite row failed. The persisted `CriterionResult.gating` field carries this to every consumer, so no reader needs the original criterion to know whether a low score mattered. -**Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` — calculated regardless for quality assessment. +**Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` over the gating criteria — calculated regardless of pass/fail for quality assessment. ### Glob patterns in `path` diff --git a/evalboard/app/runs/[id]/[...task]/_chips.tsx b/evalboard/app/runs/[id]/[...task]/_chips.tsx index 09e1c44c5..b5f01d9be 100644 --- a/evalboard/app/runs/[id]/[...task]/_chips.tsx +++ b/evalboard/app/runs/[id]/[...task]/_chips.tsx @@ -26,8 +26,9 @@ export function Expandable({ ); } -// `gating: false` (weight: 0) criteria are informational — they cannot fail the -// task, so rendering PASS/FAIL for them would contradict the task's own status. +// `gating: false` (weight: 0 or report_only) criteria are informational — they +// cannot fail the task, so rendering PASS/FAIL for them would contradict the +// task's own status. export function ResultPill({ passed, gating = true, diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index feb7d7c86..776c48709 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -152,9 +152,9 @@ export interface CriterionResult { details: string | null; error: string | null; evaluationStatus: "evaluated" | "not_evaluated"; - // Mirrors the Python CriterionResult fields. `gating: false` (weight: 0) means - // the criterion is informational — measured, but excluded from the score and - // the pass/fail gate, so it must not render as PASS/FAIL. + // Mirrors the Python CriterionResult fields. `gating: false` (weight: 0 or + // report_only) means the criterion is informational — measured, but excluded + // from the score and the pass/fail gate, so it must not render as PASS/FAIL. passThreshold: number; gating: boolean; } diff --git a/plugins/coder-eval/reference/criteria.md b/plugins/coder-eval/reference/criteria.md index b5f34993e..b0473db98 100644 --- a/plugins/coder-eval/reference/criteria.md +++ b/plugins/coder-eval/reference/criteria.md @@ -19,7 +19,8 @@ Optional: | Field | What it is | | --- | --- | -| `weight` | Relative importance of this criterion in the weighted score (default: 1.0). Set to 0 to make the criterion purely INFORMATIONAL -- useful for side-effect checks (e.g. a setup command): it is excluded from the weighted score AND from the pass/fail gate, so scoring below its pass_threshold no longer flips the task to FAILURE. The result is still computed, stored, and rendered in reports. A weight=0 criterion may not set a stop_early block or suite_thresholds (arming a non-gating criterion for a pass/fail gate is incoherent). | +| `weight` | Relative importance of this criterion in the weighted score (default: 1.0). Set to 0 to make the criterion purely INFORMATIONAL -- useful for side-effect checks (e.g. a setup command): it is excluded from the weighted score AND from the pass/fail gate, so scoring below its pass_threshold no longer flips the task to FAILURE. The result is still computed, stored, and rendered in reports. A weight=0 criterion may not set a stop_early block or suite_thresholds (arming a non-gating criterion for a pass/fail gate is incoherent). To keep a non-zero declared weight on an informational criterion, set report_only: true instead. | +| `report_only` | Make the criterion INFORMATIONAL without rewriting its declared weight (default: false). A report_only criterion still runs, and its raw score, weight and pass_threshold are stored and rendered, but it is excluded from the weighted score AND from the pass/fail gate, exactly like weight=0. Use it when the declared weight must stay for provenance or parity with another evaluation contract. Like weight=0, it may not set a stop_early block or suite_thresholds. | | `pass_threshold` | Minimum score required to pass (default: 0.9 = 90%) | | `suite_thresholds` | Across-row thresholds as {metric_name: minimum}. Only valid on tasks that declare a dataset:. The criterion passes at the suite level iff every listed metric meets its minimum. Metric names come from the criterion's aggregate() output (e.g. 'accuracy', 'f1.macro', 'recall.positive' for classification_match). | diff --git a/plugins/coder-eval/skills/lint-tasks/SKILL.md b/plugins/coder-eval/skills/lint-tasks/SKILL.md index 93988af8c..4b46e3560 100644 --- a/plugins/coder-eval/skills/lint-tasks/SKILL.md +++ b/plugins/coder-eval/skills/lint-tasks/SKILL.md @@ -110,15 +110,17 @@ chosen. - **low** — polish. Naming, tags, a description that undersells what the task does. **Gameability findings must quote the weight at risk.** This is computable from the YAML -alone, so compute it: total the `weight` of every criterion the cheap path would satisfy and -compare it to the task's total weight. *"A single `--file` call satisfies 14.0 of 33.0 -weight"* is verifiable and actionable; *"High — loose pattern"* is neither. +alone, so compute it: total the `weight` of every scoring criterion the cheap path would +satisfy and compare it to the task's total scoring weight. A `report_only: true` criterion +is not scoring: leave its weight out of both totals. *"A single `--file` call satisfies 14.0 +of 33.0 weight"* is verifiable and actionable; *"High — loose pattern"* is neither. Then map it, in this order: -1. Does the cheap path satisfy **every** scoring criterion (`weight` above 0)? A task passes - only when each of those meets its own `pass_threshold` — there is no task-level weighted - gate — so if the cheap path clears all of them, the task **cannot fail**: `critical`. +1. Does the cheap path satisfy **every** scoring criterion (`weight` above 0, no + `report_only: true`)? A task passes only when each of those meets its own + `pass_threshold` — there is no task-level weighted gate — so if the cheap path clears + all of them, the task **cannot fail**: `critical`. *Exception:* if any criterion carries a `stop_early:` block, a run the watcher actually cuts short is gated on the **armed subset only**, weighted against `stop_early_gate_threshold`. That gate is narrower than the full set, so the cheap path buys *more* there, not less — diff --git a/plugins/coder-eval/skills/task/SKILL.md b/plugins/coder-eval/skills/task/SKILL.md index b4800aad6..8d050dd3c 100644 --- a/plugins/coder-eval/skills/task/SKILL.md +++ b/plugins/coder-eval/skills/task/SKILL.md @@ -127,7 +127,7 @@ Rules that matter: live service — nothing verifies the claim, and the command runs unchanged. - `weight` reflects importance: `0.5` nice-to-have, `1.0` standard, `1.5`–`2.0` critical. `weight: 0` makes a criterion informational (reported, but excluded from the score and - the pass/fail gate). + the pass/fail gate). `report_only: true` does the same and keeps the declared weight. - The default `pass_threshold: 0.9` is right for most criteria; use `1.0` only for binary checks. - Omit the `agent:` block unless the task needs non-default settings. Agent config is diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 1ff05d3a6..115cf9edf 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -555,7 +555,7 @@ def _report_and_exit( for criterion, cr in zip(task.success_criteria, criteria_results, strict=True): if not criterion.is_gating: - # weight=0 is informational: it cannot pass/fail the task, so don't + # An informational criterion cannot pass/fail the task, so don't # render it as ✓/✗ (that would contradict the gate and the exit code). status = "[dim]○[/dim]" else: @@ -569,7 +569,7 @@ def _report_and_exit( console.print(f" [red]Error: {cr.error}[/red]") console.print() - # Gate over gating criteria only (weight=0 is informational and cannot fail + # Gate over gating criteria only (an informational one cannot fail # the task) so this summary + the exit code below match final_status. gating = [(cr, c) for cr, c in zip(criteria_results, task.success_criteria, strict=True) if c.is_gating] passed = sum(1 for cr, c in gating if cr.score >= c.pass_threshold) @@ -581,7 +581,7 @@ def _report_and_exit( console.print(f" Passed: {passed}/{total}") console.print(f" Failed: {failed}/{total}") if informational: - console.print(f" [dim]Informational (weight=0, not gated): {informational}[/dim]") + console.print(f" [dim]Informational (not gated): {informational}[/dim]") console.print(f"\n[dim]Run directory: {prepared_run_dir}[/dim]") if result.sandbox_path: console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]") diff --git a/src/coder_eval/evaluation/checker.py b/src/coder_eval/evaluation/checker.py index bb07690c8..0d52a3f1f 100644 --- a/src/coder_eval/evaluation/checker.py +++ b/src/coder_eval/evaluation/checker.py @@ -267,7 +267,7 @@ def _finalize_result(self, criterion: SuccessCriterion, result: CriterionResult) logger.debug(f"Criterion '{criterion_type}' score: {result.score:.2f}") if result.score < criterion.pass_threshold: - # An informational (weight: 0) criterion cannot fail the task, so + # An informational criterion cannot fail the task, so # logging it as FAILED contradicts the final status. Say what it is. logger.info( "Criterion '%s' %s (score=%.2f, threshold=%.2f): %s", diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index e460e3d3e..e21230765 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -118,7 +118,20 @@ class BaseSuccessCriterion(BaseModel, ABC): "below its pass_threshold no longer flips the task to FAILURE. The result is still " "computed, stored, and rendered in reports. A weight=0 criterion may not set a stop_early " "block or suite_thresholds (arming a non-gating criterion for a pass/fail gate is " - "incoherent)." + "incoherent). To keep a non-zero declared weight on an informational criterion, set " + "report_only: true instead." + ), + ) + + report_only: bool = Field( + default=False, + description=( + "Make the criterion INFORMATIONAL without rewriting its declared weight (default: false). " + "A report_only criterion still runs, and its raw score, weight and pass_threshold are " + "stored and rendered, but it is excluded from the weighted score AND from the pass/fail " + "gate, exactly like weight=0. Use it when the declared weight must stay for " + "provenance or parity with another evaluation contract. Like weight=0, it may not set a " + "stop_early block or suite_thresholds." ), ) @@ -174,43 +187,48 @@ def model_post_init(self, context: Any, /) -> None: self.__pydantic_fields_set__.add("type") @model_validator(mode="after") - def check_weight_zero_is_not_gating(self) -> Self: - """Reject ``weight: 0`` combined with any gate-arming field. - - ``weight: 0`` makes a criterion informational — excluded from the score - and from the pass/fail gate (see ``is_gating``). Both the early-stop - triggers (``is_stop_armed``: arms the per-row early-stop gate) and - ``suite_thresholds`` (arms the across-row suite gate, which drives the - run's exit code) would let an "informational" criterion flip a run to - failure — directly contradicting the field's contract. So both - combinations are authoring errors, caught at load time rather than - surfacing as a confusing exit code later. + def check_informational_is_not_armed(self) -> Self: + """Reject an informational criterion combined with any gate-arming field. + + ``weight: 0`` and ``report_only: true`` make a criterion informational — + excluded from the score and from the pass/fail gate (see ``is_gating``). + Both the early-stop triggers (``is_stop_armed``: arms the per-row + early-stop gate) and ``suite_thresholds`` (arms the across-row suite + gate, which drives the run's exit code) would let an "informational" + criterion flip a run to failure — directly contradicting the field's + contract. So both combinations are authoring errors, caught at load time + rather than surfacing as a confusing exit code later. """ - if self.weight == 0.0: - if self.is_stop_armed: - raise ValueError( - f"criterion {self.type!r}: weight=0 makes the criterion informational (non-gating), " - + "so it cannot also set a stop_early block, which arms it for a pass/fail gate. " - + "Give it a non-zero weight, or drop the block." - ) - if self.suite_thresholds is not None: - raise ValueError( - f"criterion {self.type!r}: weight=0 makes the criterion informational (non-gating), " - + "so it cannot also set suite_thresholds (which arms it for a pass/fail gate). " - + "Give it a non-zero weight, or drop suite_thresholds." - ) + if self.is_gating: + return self + if self.report_only: + cause, remedy = "report_only: true", "Drop report_only" + else: + cause, remedy = "weight=0", "Give it a non-zero weight" + if self.is_stop_armed: + raise ValueError( + f"criterion {self.type!r}: {cause} makes the criterion informational (non-gating), " + + "so it cannot also set a stop_early block, which arms it for a pass/fail gate. " + + f"{remedy}, or drop the block." + ) + if self.suite_thresholds is not None: + raise ValueError( + f"criterion {self.type!r}: {cause} makes the criterion informational (non-gating), " + + "so it cannot also set suite_thresholds (which arms it for a pass/fail gate). " + + f"{remedy}, or drop suite_thresholds." + ) return self @property def is_gating(self) -> bool: """True when this criterion participates in the task's pass/fail gate. - Single source of truth for "does a low score here fail the task". A - ``weight: 0`` criterion is informational: it is excluded from the - weighted score (it contributes 0 to both numerator and denominator) and, - by the same token, from the gate — so the two never disagree. + Single source of truth for "does a low score here fail the task", read by + both the gate and ``calculate_weighted_score`` so the two never disagree. + A ``weight: 0`` or ``report_only: true`` criterion is informational: it + is excluded from the weighted score and from the gate. """ - return self.weight > 0.0 + return self.weight > 0.0 and not self.report_only # Business logic (check operations) moved to SuccessChecker in evaluator.py diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index c8c1a8933..f93b05dc0 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -100,7 +100,7 @@ class CriterionResult(BaseModel): default=True, description=( "Whether a below-threshold score here fails the task (mirrors " - "BaseSuccessCriterion.is_gating, i.e. weight > 0). False marks an informational " + "BaseSuccessCriterion.is_gating: weight > 0 and not report_only). False marks an informational " "criterion: it is measured and reported but excluded from the score and the " "pass/fail gate, so every display surface must render it as informational rather " "than failed. Defaults True so a result built without a source criterion reads " @@ -739,7 +739,11 @@ def has_execution_failure(self) -> bool: ) def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None: - """Calculate weighted average score from criterion results. + """Calculate weighted average score from the GATING criterion results. + + Informational criteria (``BaseSuccessCriterion.is_gating`` is False: + ``weight: 0`` or ``report_only: true``) contribute to neither term, so + the score covers exactly the criteria the gate reads. Args: criteria: Original criterion definitions with weights @@ -761,6 +765,8 @@ def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None: total_weight = 0.0 for result, criterion in zip(self.success_criteria_results, criteria, strict=True): + if not criterion.is_gating: + continue total_weighted_score += result.score * criterion.weight total_weight += criterion.weight @@ -769,12 +775,13 @@ def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None: def all_criteria_passed(self, criteria: list[SuccessCriterion]) -> bool: """True iff every GATING criterion result meets its pass_threshold. - Single source of truth for the success gate. ``weight: 0`` criteria are - informational (``BaseSuccessCriterion.is_gating`` is False): they are + Single source of truth for the success gate. ``weight: 0`` and + ``report_only: true`` criteria are informational + (``BaseSuccessCriterion.is_gating`` is False): they are excluded from the weighted score, so they are excluded from the gate too — otherwise a criterion that contributes nothing to the score could still single-handedly flip the task to FAILURE. A task whose criteria - are ALL weight-0 has an empty gate and therefore passes. + are ALL informational has an empty gate and therefore passes. A results/criteria length mismatch raises ``ValueError`` rather than silently truncating — the ``len()`` pre-check is required because @@ -824,7 +831,7 @@ def armed_criteria_passed( if total_weight <= 0.0: # HAZARD: unreachable today, but a defensive guard on a pass/fail gate # must fail CLOSED against a future subclass that bypasses the - # weight=0 validator. + # informational-criterion validator. return False weighted_score = sum((1.0 if r.score >= c.pass_threshold else 0.0) * c.weight for r, c in armed) / total_weight return weighted_score >= gate_threshold diff --git a/src/coder_eval/reports/html.py b/src/coder_eval/reports/html.py index 03b1a1d74..370cfd4f2 100644 --- a/src/coder_eval/reports/html.py +++ b/src/coder_eval/reports/html.py @@ -585,9 +585,9 @@ def _render_criteria( if not evaluated: advisory = '