From 0c270d1b9dc757064337618704527a7ea08ed613 Mon Sep 17 00:00:00 2001 From: JYunth Date: Tue, 10 Nov 2026 18:19:15 +0530 Subject: [PATCH 1/2] fix(criteria): count unreadable classification_match rows as misses classification_match only caught FileNotFoundError when reading the label file. Any other read error (a glob matching more than one file, a file that is not UTF-8, a directory) came back as a plain CriterionResult, which the classification overlay skips. The row dropped out of accuracy, P/R/F1 and the confusion matrix, so 2 correct rows out of 3 reported accuracy 1.0 and passed an accuracy gate. Record those rows as (other), with the read error in details, so they stay in the rollup as a miss. Fixes #253 --- docs/TASK_DEFINITION_GUIDE.md | 2 +- .../criteria/classification_match.py | 6 ++- src/coder_eval/models/criteria.py | 2 +- tests/test_classification_match.py | 42 +++++++++++++++++-- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 12a572a52..6e5e18693 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1349,7 +1349,7 @@ The judge runs with the evaluator's API credentials and can execute arbitrary Ba Matches a single label the agent wrote to a file against ground truth — the file-based classifier. Reads the file, normalizes the content (strip, and lowercase unless `case_sensitive`), and compares it to `expected_label`. **Binary scoring:** `1.0` on a match, else `0.0`. -The observed label is the canonical form from `allowed_labels` when the content matches; otherwise `(none)` when the file is missing/empty and `(other)` when the content isn't in the allowed set. Both sentinels are recorded so a suite rollup shows them as real failure classes in the confusion matrix. +The observed label is the canonical form from `allowed_labels` when the content matches; otherwise `(none)` when the file is missing/empty and `(other)` when the content isn't in the allowed set or the file can't be read (for example, it isn't UTF-8, or a glob matches more than one file). Both sentinels are recorded so a suite rollup shows them as real failure classes in the confusion matrix. ```yaml - type: "classification_match" diff --git a/src/coder_eval/criteria/classification_match.py b/src/coder_eval/criteria/classification_match.py index 52261a26a..30f9817ca 100644 --- a/src/coder_eval/criteria/classification_match.py +++ b/src/coder_eval/criteria/classification_match.py @@ -50,11 +50,15 @@ def _check_impl( context: CheckContext | None = None, ) -> CriterionResult: expected = self._canonicalize(criterion.expected_label, criterion) + unreadable = "" try: raw = sandbox.get_file_content(criterion.path) except FileNotFoundError: observed = _SENTINEL_NONE + except (OSError, ValueError) as e: + observed = _SENTINEL_OTHER + unreadable = f" (unreadable: {type(e).__name__}: {e})" else: content = raw.strip() observed = self._canonicalize(content, criterion) if content else _SENTINEL_NONE @@ -63,7 +67,7 @@ def _check_impl( observed = _SENTINEL_OTHER score = 1.0 if observed == expected else 0.0 - details = f"observed={observed!r}, expected={expected!r}" + details = f"observed={observed!r}, expected={expected!r}{unreadable}" return ClassificationCriterionResult( criterion_type=criterion.type, diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 1e661aa73..b1b58948d 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -987,7 +987,7 @@ class ClassificationMatchCriterion(BaseSuccessCriterion): compares it to ``expected_label``. The observed label is the canonical form from ``allowed_labels`` when it matches; otherwise ``'(none)'`` when the file is missing / empty and ``'(other)'`` when the content is not in the - allowed set. Both are recorded as sentinels so the suite rollup can show + allowed set or the file cannot be read. Both are recorded as sentinels so the suite rollup can show them as real failure classes in the confusion matrix. The checker returns a ``ClassificationCriterionResult`` (subclass of diff --git a/tests/test_classification_match.py b/tests/test_classification_match.py index f3efced48..99e1d28b8 100644 --- a/tests/test_classification_match.py +++ b/tests/test_classification_match.py @@ -26,13 +26,16 @@ class _FakeSandbox: """Minimal sandbox surface needed by ClassificationMatchChecker.get_file_content.""" - def __init__(self, files: dict[str, str]): + def __init__(self, files: dict[str, str | Exception]): self._files = files def get_file_content(self, path: str) -> str: if path not in self._files: raise FileNotFoundError(path) - return self._files[path] + content = self._files[path] + if isinstance(content, Exception): + raise content + return content def _check( @@ -40,7 +43,7 @@ def _check( path: str, expected_label: str, allowed_labels: list[str], - files: dict[str, str], + files: dict[str, str | Exception], case_sensitive: bool = False, ) -> ClassificationCriterionResult: crit = ClassificationMatchCriterion( @@ -150,6 +153,25 @@ def test_empty_file_becomes_none_sentinel(self) -> None: assert result.score == 0.0 assert result.observed_label == "(none)" + @pytest.mark.parametrize( + "error", + [ + ValueError("Pattern 'out/*.txt' matches 2 files — refusing to guess which to grade"), + UnicodeDecodeError("utf-8", b"\xff\xfe", 0, 1, "invalid start byte"), + IsADirectoryError("result.txt"), + ], + ) + def test_unreadable_file_becomes_other_sentinel(self, error: Exception) -> None: + result = _check( + path="result.txt", + expected_label="positive", + allowed_labels=["positive", "negative"], + files={"result.txt": error}, + ) + assert result.score == 0.0 + assert result.observed_label == "(other)" + assert type(error).__name__ in (result.details or "") + class TestClassificationMatchAggregate: """Across-row aggregate: accuracy / recall / F1 / confusion.""" @@ -352,6 +374,20 @@ def test_thresholds_one_fails(self, tmp_path: Path) -> None: assert by_metric["recall.negative"].passed is False assert by_metric["recall.negative"].actual_value == pytest.approx(0.5) + def test_unreadable_row_counts_as_a_miss(self, tmp_path: Path) -> None: + crit = self._criterion(suite_thresholds={"accuracy": 0.9}) + checker = ClassificationMatchChecker() + rows = self._mislabeled_rows() + unreadable = _FakeSandbox({"result.txt": ValueError("Pattern 'result.txt' matches 2 files")}) + r3_crit = crit.model_copy(update={"expected_label": "negative"}) + rows[2].result.success_criteria_results = [checker.check(r3_crit, unreadable)] # type: ignore[arg-type] + + rollup = _compute_suite_rollup("s", "v1", rows, tmp_path, task_criteria=[crit]) + agg = rollup.criterion_aggregates[0] + assert agg.metrics["accuracy"] == pytest.approx(2 / 3) + assert agg.details["total_pairs"] == 3 + assert agg.passed is False + def test_threshold_on_missing_metric_fails(self, tmp_path: Path) -> None: rows = self._mislabeled_rows() crit = self._criterion(suite_thresholds={"does_not_exist": 0.5}) From 3ade4d257b452756617eca900861b1817a519765 Mon Sep 17 00:00:00 2001 From: JYunth Date: Wed, 11 Nov 2026 11:00:23 +0530 Subject: [PATCH 2/2] fix(criteria): keep the read error on unreadable classification_match rows Review follow-up. An unreadable label file now sets `error` on the ClassificationCriterionResult as well as recording `(other)`, so the row stays in accuracy/P/R/F1 and still counts in criterion_stats.error_count and the error badges, as it did before this PR. Also adds a real-Sandbox test (ambiguous glob, non-UTF-8 file, directory), a PermissionError case, a test that CheckerMisuseError still propagates, and an error_count and confusion-cell check in the rollup test. Notes the CE039 decision at the handler, re-wraps the docstring, and updates the allowed_labels description and the generated plugin reference. --- plugins/coder-eval/reference/criteria.md | 2 +- .../criteria/classification_match.py | 10 ++-- src/coder_eval/models/criteria.py | 9 ++-- tests/test_classification_match.py | 48 ++++++++++++++++++- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/plugins/coder-eval/reference/criteria.md b/plugins/coder-eval/reference/criteria.md index 526e1fbdd..3a15fb1fd 100644 --- a/plugins/coder-eval/reference/criteria.md +++ b/plugins/coder-eval/reference/criteria.md @@ -70,7 +70,7 @@ Match a single label written by the agent to a file against ground truth. | --- | --- | | `path` | Path to the file (relative to sandbox) containing the agent's predicted label; may be a glob matching exactly one file | | `expected_label` | Ground-truth label for this row | -| `allowed_labels` | Canonical label set. File content not in this set is treated as '(other)'. | +| `allowed_labels` | Canonical label set. File content not in this set, or a file that cannot be read, is treated as '(other)'. | Optional: diff --git a/src/coder_eval/criteria/classification_match.py b/src/coder_eval/criteria/classification_match.py index 30f9817ca..23cc368af 100644 --- a/src/coder_eval/criteria/classification_match.py +++ b/src/coder_eval/criteria/classification_match.py @@ -50,15 +50,16 @@ def _check_impl( context: CheckContext | None = None, ) -> CriterionResult: expected = self._canonicalize(criterion.expected_label, criterion) - unreadable = "" + read_error: str | None = None try: raw = sandbox.get_file_content(criterion.path) except FileNotFoundError: observed = _SENTINEL_NONE except (OSError, ValueError) as e: + # CE039: a miss that stays in the rollup; `error` keeps the cause countable. observed = _SENTINEL_OTHER - unreadable = f" (unreadable: {type(e).__name__}: {e})" + read_error = f"{type(e).__name__}: {e}" else: content = raw.strip() observed = self._canonicalize(content, criterion) if content else _SENTINEL_NONE @@ -67,13 +68,16 @@ def _check_impl( observed = _SENTINEL_OTHER score = 1.0 if observed == expected else 0.0 - details = f"observed={observed!r}, expected={expected!r}{unreadable}" + details = f"observed={observed!r}, expected={expected!r}" + if read_error is not None: + details += f" (unreadable: {read_error})" return ClassificationCriterionResult( criterion_type=criterion.type, description=criterion.description, score=score, details=details, + error=read_error, observed_label=observed, expected_label=expected, ) diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index b1b58948d..fbca08967 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -987,8 +987,9 @@ class ClassificationMatchCriterion(BaseSuccessCriterion): compares it to ``expected_label``. The observed label is the canonical form from ``allowed_labels`` when it matches; otherwise ``'(none)'`` when the file is missing / empty and ``'(other)'`` when the content is not in the - allowed set or the file cannot be read. Both are recorded as sentinels so the suite rollup can show - them as real failure classes in the confusion matrix. + allowed set or the file cannot be read. Both are recorded as sentinels + so the suite rollup can show them as real failure classes in the + confusion matrix. The checker returns a ``ClassificationCriterionResult`` (subclass of ``CriterionResult``) carrying observed + expected labels, which the @@ -1015,7 +1016,9 @@ class ClassificationMatchCriterion(BaseSuccessCriterion): expected_label: str = Field(description="Ground-truth label for this row") allowed_labels: list[str] = Field( min_length=1, - description="Canonical label set. File content not in this set is treated as '(other)'.", + description=( + "Canonical label set. File content not in this set, or a file that cannot be read, is treated as '(other)'." + ), ) case_sensitive: bool = Field( default=False, diff --git a/tests/test_classification_match.py b/tests/test_classification_match.py index 99e1d28b8..94f657de9 100644 --- a/tests/test_classification_match.py +++ b/tests/test_classification_match.py @@ -10,6 +10,7 @@ import yaml from coder_eval.criteria.classification_match import ClassificationMatchChecker +from coder_eval.errors import CheckerMisuseError from coder_eval.models import ( AgentKind, ClassificationCriterionResult, @@ -17,10 +18,12 @@ CriterionResult, EvaluationResult, FinalStatus, + SandboxConfig, TaskDefinition, TaskResult, ) from coder_eval.reports.markdown import _compute_suite_rollup, _render_suite_markdown +from coder_eval.sandbox import Sandbox class _FakeSandbox: @@ -159,6 +162,7 @@ def test_empty_file_becomes_none_sentinel(self) -> None: ValueError("Pattern 'out/*.txt' matches 2 files — refusing to guess which to grade"), UnicodeDecodeError("utf-8", b"\xff\xfe", 0, 1, "invalid start byte"), IsADirectoryError("result.txt"), + PermissionError("result.txt"), ], ) def test_unreadable_file_becomes_other_sentinel(self, error: Exception) -> None: @@ -170,7 +174,46 @@ def test_unreadable_file_becomes_other_sentinel(self, error: Exception) -> None: ) assert result.score == 0.0 assert result.observed_label == "(other)" - assert type(error).__name__ in (result.details or "") + assert result.error is not None + assert result.error.startswith(f"{type(error).__name__}: ") + + def test_checker_misuse_still_escalates(self) -> None: + crit = ClassificationMatchCriterion( + description="label match", + path="../outside.txt", + expected_label="positive", + allowed_labels=["positive", "negative"], + ) + sandbox = _FakeSandbox({"../outside.txt": CheckerMisuseError("path escapes the sandbox")}) + with pytest.raises(CheckerMisuseError): + ClassificationMatchChecker().check(crit, sandbox) # type: ignore[arg-type] + + @pytest.mark.parametrize("case", ["ambiguous_glob", "not_utf8", "directory"]) + def test_unreadable_file_in_a_real_sandbox(self, tmp_path: Path, case: str) -> None: + work = tmp_path / "work" + (work / "out").mkdir(parents=True) + if case == "ambiguous_glob": + (work / "out" / "a.txt").write_text("positive") + (work / "out" / "b.txt").write_text("positive") + elif case == "not_utf8": + (work / "out" / "a.txt").write_bytes(b"\xff\xfe") + else: + (work / "out" / "a.txt").mkdir() + sandbox = Sandbox(SandboxConfig(driver="tempdir"), task_id="t") + sandbox.sandbox_dir = work + crit = ClassificationMatchCriterion( + description="label match", + path="out/*.txt" if case == "ambiguous_glob" else "out/a.txt", + expected_label="negative", + allowed_labels=["positive", "negative"], + ) + + result = ClassificationMatchChecker().check(crit, sandbox) + + assert isinstance(result, ClassificationCriterionResult) + assert result.observed_label == "(other)" + assert result.score == 0.0 + assert result.error is not None class TestClassificationMatchAggregate: @@ -386,7 +429,10 @@ def test_unreadable_row_counts_as_a_miss(self, tmp_path: Path) -> None: agg = rollup.criterion_aggregates[0] assert agg.metrics["accuracy"] == pytest.approx(2 / 3) assert agg.details["total_pairs"] == 3 + assert {"expected": "negative", "observed": "(other)", "count": 1} in agg.details["confusion"] assert agg.passed is False + stats = next(cs for cs in rollup.criterion_stats if cs.criterion_type == "classification_match") + assert stats.error_count == 1 def test_threshold_on_missing_metric_fails(self, tmp_path: Path) -> None: rows = self._mislabeled_rows()