diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index bef3866db..0584b7ac5 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -170,6 +170,7 @@ class MyCriterion(BaseSuccessCriterion): Union membership is required — a run validates that every union member's `type` has a registered checker, and rejects unknown `type` tags in YAML. `BaseSuccessCriterion` gives you `description`, `weight` (default 1.0; `0` = informational/non-gating), +`report_only` (default `false`; `true` = informational at the declared weight), `pass_threshold` (default 0.9) and `suite_thresholds` for free, with `extra="forbid"` so YAML typos are caught. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index c1e112206..88af88a5d 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -211,9 +211,9 @@ A discriminated union on `result_kind` (`basic` / `judge` / `classification`). B `criterion_type`, `description`, `score` (0.0–1.0), `details`, `error`, `evaluation_status` (`evaluated` by default; `not_evaluated` means the check did not run and is distinct from an evaluated 0.0), -`pass_threshold` (default 0.9), `gating` (default `true`; `false` = informational / -weight-0, excluded from the score and the pass/fail gate). The base allows extra -fields so subclass keys round-trip. +`pass_threshold` (default 0.9), `gating` (default `true`; `false` = informational, from +`weight: 0` or `report_only: true`, excluded from the score and the pass/fail gate). The +base allows extra fields so subclass keys round-trip. - **`classification`** adds `observed_label`, `expected_label` (sentinels like `(none)` / `(other)` allowed). Emitted by `classification_match`, `skill_triggered`. diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 8c9481567..363c9c4fa 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -749,6 +749,7 @@ All criteria share these fields: |-------|---------|-------------| | `description` | — | Human-readable description (required) | | `weight` | 1.0 | Relative importance for weighted score. `0` = **informational**: excluded from both the score and the pass/fail gate | +| `report_only` | `false` | `true` = **informational** without changing the declared `weight`: excluded from both the score and the pass/fail gate, like `weight: 0` | | `pass_threshold` | 0.9 | Minimum score (0.0–1.0) to pass | | `stop_early` | `null` | **Only on live-observable criteria** (`skill_triggered`, `command_executed`). Presence arms the criterion for early stop (no run-level switch needed): an effective fail may end the run (weighted ceiling rule, recall deferral). Keys: `on_pass: stop\|continue` (default `continue`), `decide_within: N` (timeout → effective fail, reported as `decision_budget_exceeded`). Inert triggers by design on instances that can't decide their polarity (dataset fan-out support). See [`stop_early`](#stop_early-opt-in-early-stop). | @@ -758,11 +759,33 @@ All criteria share these fields: - **Continuous** (0.0–1.0): `reference_comparison`, `commands_efficiency`, `llm_judge`, `agent_judge`, `system_one_judge` **Task success:** all *gating* criteria must score >= their `pass_threshold`. A -criterion with `weight: 0` is informational — it is still checked, stored, and -rendered in reports, but it neither contributes to the score nor fails the task. -(A `weight: 0` criterion may not set a `stop_early` block or `suite_thresholds`: arming a -non-gating criterion for the early-stop or suite gate would let an -"informational" check flip a run to failure.) +criterion with `weight: 0` or `report_only: true` is informational — it is still +checked, stored, and rendered in reports, but it neither contributes to the score nor +fails the task. (An informational criterion may not set a `stop_early` block or +`suite_thresholds`: arming a non-gating criterion for the early-stop or suite gate would +let an "informational" check flip a run to failure.) + +Use `report_only: true` when the criterion must keep its declared `weight` — for example, +to match the weights of another evaluation contract. A `pass_threshold: 0.0` is not the +same thing: that criterion always passes the gate, but its score and weight still count +in `weighted_score`. + +```yaml +success_criteria: + - type: run_command + description: Unit tests pass + command: pytest -q + weight: 8 + pass_threshold: 1.0 + - type: run_command + description: Linter is clean (recorded, not scored) + command: ruff check . + weight: 1 + report_only: true +``` + +When the tests pass, a failed `ruff check` leaves the task at `SUCCESS` with +`weighted_score: 1.0` (not 8/9), and its result is stored with `gating: false`. Every surface labels it as such rather than as a failure: the terminal shows `○` instead of `✓`/`✗`, the HTML report tags the row "informational — not gated" and @@ -772,7 +795,7 @@ suite row failed. The persisted `CriterionResult.gating` field carries this to every consumer, so no reader needs the original criterion to know whether a low score mattered. -**Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` — calculated regardless for quality assessment. +**Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` over the gating criteria — calculated regardless of pass/fail for quality assessment. ### Glob patterns in `path` diff --git a/evalboard/app/runs/[id]/[...task]/_chips.tsx b/evalboard/app/runs/[id]/[...task]/_chips.tsx index 09e1c44c5..b5f01d9be 100644 --- a/evalboard/app/runs/[id]/[...task]/_chips.tsx +++ b/evalboard/app/runs/[id]/[...task]/_chips.tsx @@ -26,8 +26,9 @@ export function Expandable({ ); } -// `gating: false` (weight: 0) criteria are informational — they cannot fail the -// task, so rendering PASS/FAIL for them would contradict the task's own status. +// `gating: false` (weight: 0 or report_only) criteria are informational — they +// cannot fail the task, so rendering PASS/FAIL for them would contradict the +// task's own status. export function ResultPill({ passed, gating = true, diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index feb7d7c86..776c48709 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -152,9 +152,9 @@ export interface CriterionResult { details: string | null; error: string | null; evaluationStatus: "evaluated" | "not_evaluated"; - // Mirrors the Python CriterionResult fields. `gating: false` (weight: 0) means - // the criterion is informational — measured, but excluded from the score and - // the pass/fail gate, so it must not render as PASS/FAIL. + // Mirrors the Python CriterionResult fields. `gating: false` (weight: 0 or + // report_only) means the criterion is informational — measured, but excluded + // from the score and the pass/fail gate, so it must not render as PASS/FAIL. passThreshold: number; gating: boolean; } diff --git a/plugins/coder-eval/reference/criteria.md b/plugins/coder-eval/reference/criteria.md index b5f34993e..b0473db98 100644 --- a/plugins/coder-eval/reference/criteria.md +++ b/plugins/coder-eval/reference/criteria.md @@ -19,7 +19,8 @@ Optional: | Field | What it is | | --- | --- | -| `weight` | Relative importance of this criterion in the weighted score (default: 1.0). Set to 0 to make the criterion purely INFORMATIONAL -- useful for side-effect checks (e.g. a setup command): it is excluded from the weighted score AND from the pass/fail gate, so scoring below its pass_threshold no longer flips the task to FAILURE. The result is still computed, stored, and rendered in reports. A weight=0 criterion may not set a stop_early block or suite_thresholds (arming a non-gating criterion for a pass/fail gate is incoherent). | +| `weight` | Relative importance of this criterion in the weighted score (default: 1.0). Set to 0 to make the criterion purely INFORMATIONAL -- useful for side-effect checks (e.g. a setup command): it is excluded from the weighted score AND from the pass/fail gate, so scoring below its pass_threshold no longer flips the task to FAILURE. The result is still computed, stored, and rendered in reports. A weight=0 criterion may not set a stop_early block or suite_thresholds (arming a non-gating criterion for a pass/fail gate is incoherent). To keep a non-zero declared weight on an informational criterion, set report_only: true instead. | +| `report_only` | Make the criterion INFORMATIONAL without rewriting its declared weight (default: false). A report_only criterion still runs, and its raw score, weight and pass_threshold are stored and rendered, but it is excluded from the weighted score AND from the pass/fail gate, exactly like weight=0. Use it when the declared weight must stay for provenance or parity with another evaluation contract. Like weight=0, it may not set a stop_early block or suite_thresholds. | | `pass_threshold` | Minimum score required to pass (default: 0.9 = 90%) | | `suite_thresholds` | Across-row thresholds as {metric_name: minimum}. Only valid on tasks that declare a dataset:. The criterion passes at the suite level iff every listed metric meets its minimum. Metric names come from the criterion's aggregate() output (e.g. 'accuracy', 'f1.macro', 'recall.positive' for classification_match). | diff --git a/plugins/coder-eval/skills/lint-tasks/SKILL.md b/plugins/coder-eval/skills/lint-tasks/SKILL.md index 93988af8c..4b46e3560 100644 --- a/plugins/coder-eval/skills/lint-tasks/SKILL.md +++ b/plugins/coder-eval/skills/lint-tasks/SKILL.md @@ -110,15 +110,17 @@ chosen. - **low** — polish. Naming, tags, a description that undersells what the task does. **Gameability findings must quote the weight at risk.** This is computable from the YAML -alone, so compute it: total the `weight` of every criterion the cheap path would satisfy and -compare it to the task's total weight. *"A single `--file` call satisfies 14.0 of 33.0 -weight"* is verifiable and actionable; *"High — loose pattern"* is neither. +alone, so compute it: total the `weight` of every scoring criterion the cheap path would +satisfy and compare it to the task's total scoring weight. A `report_only: true` criterion +is not scoring: leave its weight out of both totals. *"A single `--file` call satisfies 14.0 +of 33.0 weight"* is verifiable and actionable; *"High — loose pattern"* is neither. Then map it, in this order: -1. Does the cheap path satisfy **every** scoring criterion (`weight` above 0)? A task passes - only when each of those meets its own `pass_threshold` — there is no task-level weighted - gate — so if the cheap path clears all of them, the task **cannot fail**: `critical`. +1. Does the cheap path satisfy **every** scoring criterion (`weight` above 0, no + `report_only: true`)? A task passes only when each of those meets its own + `pass_threshold` — there is no task-level weighted gate — so if the cheap path clears + all of them, the task **cannot fail**: `critical`. *Exception:* if any criterion carries a `stop_early:` block, a run the watcher actually cuts short is gated on the **armed subset only**, weighted against `stop_early_gate_threshold`. That gate is narrower than the full set, so the cheap path buys *more* there, not less — diff --git a/plugins/coder-eval/skills/task/SKILL.md b/plugins/coder-eval/skills/task/SKILL.md index b4800aad6..8d050dd3c 100644 --- a/plugins/coder-eval/skills/task/SKILL.md +++ b/plugins/coder-eval/skills/task/SKILL.md @@ -127,7 +127,7 @@ Rules that matter: live service — nothing verifies the claim, and the command runs unchanged. - `weight` reflects importance: `0.5` nice-to-have, `1.0` standard, `1.5`–`2.0` critical. `weight: 0` makes a criterion informational (reported, but excluded from the score and - the pass/fail gate). + the pass/fail gate). `report_only: true` does the same and keeps the declared weight. - The default `pass_threshold: 0.9` is right for most criteria; use `1.0` only for binary checks. - Omit the `agent:` block unless the task needs non-default settings. Agent config is diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 1ff05d3a6..115cf9edf 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -555,7 +555,7 @@ def _report_and_exit( for criterion, cr in zip(task.success_criteria, criteria_results, strict=True): if not criterion.is_gating: - # weight=0 is informational: it cannot pass/fail the task, so don't + # An informational criterion cannot pass/fail the task, so don't # render it as ✓/✗ (that would contradict the gate and the exit code). status = "[dim]○[/dim]" else: @@ -569,7 +569,7 @@ def _report_and_exit( console.print(f" [red]Error: {cr.error}[/red]") console.print() - # Gate over gating criteria only (weight=0 is informational and cannot fail + # Gate over gating criteria only (an informational one cannot fail # the task) so this summary + the exit code below match final_status. gating = [(cr, c) for cr, c in zip(criteria_results, task.success_criteria, strict=True) if c.is_gating] passed = sum(1 for cr, c in gating if cr.score >= c.pass_threshold) @@ -581,7 +581,7 @@ def _report_and_exit( console.print(f" Passed: {passed}/{total}") console.print(f" Failed: {failed}/{total}") if informational: - console.print(f" [dim]Informational (weight=0, not gated): {informational}[/dim]") + console.print(f" [dim]Informational (not gated): {informational}[/dim]") console.print(f"\n[dim]Run directory: {prepared_run_dir}[/dim]") if result.sandbox_path: console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]") diff --git a/src/coder_eval/evaluation/checker.py b/src/coder_eval/evaluation/checker.py index bb07690c8..0d52a3f1f 100644 --- a/src/coder_eval/evaluation/checker.py +++ b/src/coder_eval/evaluation/checker.py @@ -267,7 +267,7 @@ def _finalize_result(self, criterion: SuccessCriterion, result: CriterionResult) logger.debug(f"Criterion '{criterion_type}' score: {result.score:.2f}") if result.score < criterion.pass_threshold: - # An informational (weight: 0) criterion cannot fail the task, so + # An informational criterion cannot fail the task, so # logging it as FAILED contradicts the final status. Say what it is. logger.info( "Criterion '%s' %s (score=%.2f, threshold=%.2f): %s", diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index e460e3d3e..e21230765 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -118,7 +118,20 @@ class BaseSuccessCriterion(BaseModel, ABC): "below its pass_threshold no longer flips the task to FAILURE. The result is still " "computed, stored, and rendered in reports. A weight=0 criterion may not set a stop_early " "block or suite_thresholds (arming a non-gating criterion for a pass/fail gate is " - "incoherent)." + "incoherent). To keep a non-zero declared weight on an informational criterion, set " + "report_only: true instead." + ), + ) + + report_only: bool = Field( + default=False, + description=( + "Make the criterion INFORMATIONAL without rewriting its declared weight (default: false). " + "A report_only criterion still runs, and its raw score, weight and pass_threshold are " + "stored and rendered, but it is excluded from the weighted score AND from the pass/fail " + "gate, exactly like weight=0. Use it when the declared weight must stay for " + "provenance or parity with another evaluation contract. Like weight=0, it may not set a " + "stop_early block or suite_thresholds." ), ) @@ -174,43 +187,48 @@ def model_post_init(self, context: Any, /) -> None: self.__pydantic_fields_set__.add("type") @model_validator(mode="after") - def check_weight_zero_is_not_gating(self) -> Self: - """Reject ``weight: 0`` combined with any gate-arming field. - - ``weight: 0`` makes a criterion informational — excluded from the score - and from the pass/fail gate (see ``is_gating``). Both the early-stop - triggers (``is_stop_armed``: arms the per-row early-stop gate) and - ``suite_thresholds`` (arms the across-row suite gate, which drives the - run's exit code) would let an "informational" criterion flip a run to - failure — directly contradicting the field's contract. So both - combinations are authoring errors, caught at load time rather than - surfacing as a confusing exit code later. + def check_informational_is_not_armed(self) -> Self: + """Reject an informational criterion combined with any gate-arming field. + + ``weight: 0`` and ``report_only: true`` make a criterion informational — + excluded from the score and from the pass/fail gate (see ``is_gating``). + Both the early-stop triggers (``is_stop_armed``: arms the per-row + early-stop gate) and ``suite_thresholds`` (arms the across-row suite + gate, which drives the run's exit code) would let an "informational" + criterion flip a run to failure — directly contradicting the field's + contract. So both combinations are authoring errors, caught at load time + rather than surfacing as a confusing exit code later. """ - if self.weight == 0.0: - if self.is_stop_armed: - raise ValueError( - f"criterion {self.type!r}: weight=0 makes the criterion informational (non-gating), " - + "so it cannot also set a stop_early block, which arms it for a pass/fail gate. " - + "Give it a non-zero weight, or drop the block." - ) - if self.suite_thresholds is not None: - raise ValueError( - f"criterion {self.type!r}: weight=0 makes the criterion informational (non-gating), " - + "so it cannot also set suite_thresholds (which arms it for a pass/fail gate). " - + "Give it a non-zero weight, or drop suite_thresholds." - ) + if self.is_gating: + return self + if self.report_only: + cause, remedy = "report_only: true", "Drop report_only" + else: + cause, remedy = "weight=0", "Give it a non-zero weight" + if self.is_stop_armed: + raise ValueError( + f"criterion {self.type!r}: {cause} makes the criterion informational (non-gating), " + + "so it cannot also set a stop_early block, which arms it for a pass/fail gate. " + + f"{remedy}, or drop the block." + ) + if self.suite_thresholds is not None: + raise ValueError( + f"criterion {self.type!r}: {cause} makes the criterion informational (non-gating), " + + "so it cannot also set suite_thresholds (which arms it for a pass/fail gate). " + + f"{remedy}, or drop suite_thresholds." + ) return self @property def is_gating(self) -> bool: """True when this criterion participates in the task's pass/fail gate. - Single source of truth for "does a low score here fail the task". A - ``weight: 0`` criterion is informational: it is excluded from the - weighted score (it contributes 0 to both numerator and denominator) and, - by the same token, from the gate — so the two never disagree. + Single source of truth for "does a low score here fail the task", read by + both the gate and ``calculate_weighted_score`` so the two never disagree. + A ``weight: 0`` or ``report_only: true`` criterion is informational: it + is excluded from the weighted score and from the gate. """ - return self.weight > 0.0 + return self.weight > 0.0 and not self.report_only # Business logic (check operations) moved to SuccessChecker in evaluator.py diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index c8c1a8933..f93b05dc0 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -100,7 +100,7 @@ class CriterionResult(BaseModel): default=True, description=( "Whether a below-threshold score here fails the task (mirrors " - "BaseSuccessCriterion.is_gating, i.e. weight > 0). False marks an informational " + "BaseSuccessCriterion.is_gating: weight > 0 and not report_only). False marks an informational " "criterion: it is measured and reported but excluded from the score and the " "pass/fail gate, so every display surface must render it as informational rather " "than failed. Defaults True so a result built without a source criterion reads " @@ -739,7 +739,11 @@ def has_execution_failure(self) -> bool: ) def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None: - """Calculate weighted average score from criterion results. + """Calculate weighted average score from the GATING criterion results. + + Informational criteria (``BaseSuccessCriterion.is_gating`` is False: + ``weight: 0`` or ``report_only: true``) contribute to neither term, so + the score covers exactly the criteria the gate reads. Args: criteria: Original criterion definitions with weights @@ -761,6 +765,8 @@ def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None: total_weight = 0.0 for result, criterion in zip(self.success_criteria_results, criteria, strict=True): + if not criterion.is_gating: + continue total_weighted_score += result.score * criterion.weight total_weight += criterion.weight @@ -769,12 +775,13 @@ def calculate_weighted_score(self, criteria: list[SuccessCriterion]) -> None: def all_criteria_passed(self, criteria: list[SuccessCriterion]) -> bool: """True iff every GATING criterion result meets its pass_threshold. - Single source of truth for the success gate. ``weight: 0`` criteria are - informational (``BaseSuccessCriterion.is_gating`` is False): they are + Single source of truth for the success gate. ``weight: 0`` and + ``report_only: true`` criteria are informational + (``BaseSuccessCriterion.is_gating`` is False): they are excluded from the weighted score, so they are excluded from the gate too — otherwise a criterion that contributes nothing to the score could still single-handedly flip the task to FAILURE. A task whose criteria - are ALL weight-0 has an empty gate and therefore passes. + are ALL informational has an empty gate and therefore passes. A results/criteria length mismatch raises ``ValueError`` rather than silently truncating — the ``len()`` pre-check is required because @@ -824,7 +831,7 @@ def armed_criteria_passed( if total_weight <= 0.0: # HAZARD: unreachable today, but a defensive guard on a pass/fail gate # must fail CLOSED against a future subclass that bypasses the - # weight=0 validator. + # informational-criterion validator. return False weighted_score = sum((1.0 if r.score >= c.pass_threshold else 0.0) * c.weight for r, c in armed) / total_weight return weighted_score >= gate_threshold diff --git a/src/coder_eval/reports/html.py b/src/coder_eval/reports/html.py index 03b1a1d74..370cfd4f2 100644 --- a/src/coder_eval/reports/html.py +++ b/src/coder_eval/reports/html.py @@ -585,9 +585,9 @@ def _render_criteria( if not evaluated: advisory = '
no verdict — the check did not run
' elif not cr.gating and not diagnostic: - # weight: 0 — measured but excluded from the gate. Saying so here keeps + # Informational — measured but excluded from the gate. Saying so here keeps # the row from reading as a failure that the header/status contradicts. - advisory = '
informational — not gated (weight: 0)
' + advisory = '
informational — not gated
' elif not diagnostic and early_stop is not None and f"{cr.criterion_type}: {cr.description}" not in armed: advisory = '
advisory — not gated (run stopped early)
' score = _score_pill(cr.score) if evaluated else 'NOT EVALUATED' diff --git a/src/coder_eval/reports/markdown.py b/src/coder_eval/reports/markdown.py index c2d6fbfc8..c60687a9d 100644 --- a/src/coder_eval/reports/markdown.py +++ b/src/coder_eval/reports/markdown.py @@ -1034,7 +1034,7 @@ def _compute_suite_rollup( break reasons: list[str] = [] for cr in row.result.success_criteria_results: - # Informational (weight: 0) criteria cannot fail a row, so they must + # Informational criteria cannot fail a row, so they must # not supply the reason the row is reported as failed. if not cr.gating: continue diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml b/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml index 2cb1dca6e..5afa2e8ba 100644 --- a/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml +++ b/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml @@ -9,11 +9,13 @@ success_criteria: - type: file_exists description: greeting.txt exists weight: 1.0 + report_only: false pass_threshold: 0.9 path: greeting.txt - type: reference_comparison description: matches the reference greeting weight: 1.0 + report_only: false pass_threshold: 0.8 agent_file: greeting.txt reference_file: greeting.txt diff --git a/tests/fixtures/tasks/test_task_report_only_criterion.yaml b/tests/fixtures/tasks/test_task_report_only_criterion.yaml new file mode 100644 index 000000000..cef0bb588 --- /dev/null +++ b/tests/fixtures/tasks/test_task_report_only_criterion.yaml @@ -0,0 +1,20 @@ +task_id: test_task_report_only +description: A weight-8 gating criterion passes; a weight-1 report_only criterion fails. +initial_prompt: Test +# agent block required by TaskDefinition schema but unused in evaluate-only mode +agent: + type: claude-code +sandbox: + driver: tempdir + python: null +success_criteria: + - type: file_exists + path: app.py + description: app.py must exist (gating) + weight: 8.0 + pass_threshold: 1.0 + - type: file_exists + path: missing.py + description: report-only side-effect check (keeps its weight, must not gate) + weight: 1.0 + report_only: true diff --git a/tests/test_evaluate_command.py b/tests/test_evaluate_command.py index b824fe00a..9412475b6 100644 --- a/tests/test_evaluate_command.py +++ b/tests/test_evaluate_command.py @@ -166,6 +166,35 @@ def test_evaluate_command_informational_criterion_does_not_fail_exit(tmp_path): assert exc_info.value.exit_code == 0 +def test_evaluate_command_report_only_criterion_is_excluded_from_gate_and_score(tmp_path): + """A failing report_only criterion keeps its declared weight but neither gates nor scores.""" + from coder_eval.cli.evaluate_command import run_evaluation + from coder_eval.models import EvaluationResult + + task_file = FIXTURES_DIR / "tasks" / "test_task_report_only_criterion.yaml" + work_dir = tmp_path / "work" + work_dir.mkdir() + (work_dir / "app.py").write_text("print('hello')") + run_dir = tmp_path / "run" + run_dir.mkdir() + + with ( + patch("coder_eval.cli.console.console.print"), + patch("coder_eval.logging_config.setup_logging"), + pytest.raises(typer.Exit) as exc_info, + ): + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) + + assert exc_info.value.exit_code == 0 + (row,) = _row_dirs(run_dir) + record = EvaluationResult.model_validate_json((row / "task.json").read_text(encoding="utf-8")) + assert record.final_status == "SUCCESS" + assert record.weighted_score == 1.0 # not 8/9: the report_only miss is out of both terms + gating, report_only = record.success_criteria_results + assert (gating.score, gating.gating) == (1.0, True) + assert (report_only.score, report_only.gating) == (0.0, False) + + def test_evaluate_command_invalid_task_file(tmp_path): """Test evaluate command with invalid task file.""" # Use non-existent task file diff --git a/tests/test_threshold_enforcement.py b/tests/test_threshold_enforcement.py index ac5d6b1c7..c25f7f775 100644 --- a/tests/test_threshold_enforcement.py +++ b/tests/test_threshold_enforcement.py @@ -155,6 +155,62 @@ def test_zero_weight_cannot_declare_suite_thresholds(self): suite_thresholds={"mean": 0.8}, ) + def test_report_only_criterion_keeps_its_weight_but_neither_scores_nor_gates(self): + """report_only excludes a non-zero-weight criterion from the score and the gate (#101).""" + criteria = [ + FileExistsCriterion(path="f1.txt", description="crit-0", weight=8.0, pass_threshold=1.0), + FileExistsCriterion(path="f2.txt", description="crit-1", weight=1.0, pass_threshold=0.9, report_only=True), + ] + result = _make_result([1.0, 0.0]) + + result.calculate_weighted_score(criteria) + assert result.weighted_score == 1.0 # not 8/9 + assert result.all_criteria_passed(criteria) + assert criteria[1].weight == 1.0 + assert not criteria[1].is_gating + + def test_report_only_does_not_rescue_a_failing_gating_criterion(self): + criteria = [ + FileExistsCriterion(path="f1.txt", description="crit-0", weight=1.0, pass_threshold=0.9), + FileExistsCriterion(path="f2.txt", description="crit-1", weight=5.0, pass_threshold=0.9, report_only=True), + ] + result = _make_result([0.5, 1.0]) + + result.calculate_weighted_score(criteria) + assert result.weighted_score == 0.5 # the report_only pass does not lift the score + assert not result.all_criteria_passed(criteria) + + def test_all_report_only_criteria_score_zero_and_leave_an_empty_gate(self): + """Same outcome as an all-weight-0 task: nothing to score, nothing to fail on.""" + criteria = [ + FileExistsCriterion(path="f1.txt", description="crit-0", report_only=True), + FileExistsCriterion(path="f2.txt", description="crit-1", report_only=True), + ] + result = _make_result([1.0, 0.0]) + + result.calculate_weighted_score(criteria) + assert result.weighted_score == 0.0 + assert result.all_criteria_passed(criteria) + + def test_report_only_cannot_be_armed_for_early_stop(self): + with pytest.raises(ValidationError, match="report_only: true makes the criterion informational"): + CommandExecutedCriterion( + description="report-only + armed", + report_only=True, + stop_early=StopEarlyPolicy(on_pass="stop"), + tool_name="Bash", + command_pattern="pytest", + ) + + def test_report_only_cannot_declare_suite_thresholds(self): + with pytest.raises(ValidationError, match="report_only: true makes the criterion informational"): + FileExistsCriterion( + path="f1.txt", + description="report-only + suite-gated", + report_only=True, + suite_thresholds={"mean": 0.8}, + ) + def test_empty_inputs_score_zero_without_raising(self): """Empty results/criteria still yield 0.0 (not a raise) and an empty gate passes.""" result = _make_result([]) @@ -183,12 +239,17 @@ def test_checker_stamps_gating_from_the_criterion(self, tmp_path): gating = checker.check(FileExistsCriterion(path="missing.txt", description="gating", weight=1.0)) informational = checker.check(FileExistsCriterion(path="missing.txt", description="informational", weight=0.0)) + report_only = checker.check( + FileExistsCriterion(path="missing.txt", description="report-only", weight=1.0, report_only=True) + ) assert gating.gating is True assert informational.gating is False - # Both genuinely scored zero — un-gating changes the label, never the measurement. + assert report_only.gating is False + # All genuinely scored zero — un-gating changes the label, never the measurement. assert gating.score == 0.0 assert informational.score == 0.0 + assert report_only.score == 0.0 def test_criterion_result_defaults_to_gating(self): """Results persisted before the field existed must read back as gating.""" @@ -210,5 +271,5 @@ def test_html_report_labels_informational_and_excludes_it_from_the_count(self): ) assert "(1/1 passed" in html # NOT 1/2 — the informational miss isn't a failure - assert "informational — not gated (weight: 0)" in html + assert "informational — not gated" in html assert "1 informational" in html