diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 12a572a52..6e5e18693 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -1349,7 +1349,7 @@ The judge runs with the evaluator's API credentials and can execute arbitrary Ba Matches a single label the agent wrote to a file against ground truth — the file-based classifier. Reads the file, normalizes the content (strip, and lowercase unless `case_sensitive`), and compares it to `expected_label`. **Binary scoring:** `1.0` on a match, else `0.0`. -The observed label is the canonical form from `allowed_labels` when the content matches; otherwise `(none)` when the file is missing/empty and `(other)` when the content isn't in the allowed set. Both sentinels are recorded so a suite rollup shows them as real failure classes in the confusion matrix. +The observed label is the canonical form from `allowed_labels` when the content matches; otherwise `(none)` when the file is missing/empty and `(other)` when the content isn't in the allowed set or the file can't be read (for example, it isn't UTF-8, or a glob matches more than one file). Both sentinels are recorded so a suite rollup shows them as real failure classes in the confusion matrix. ```yaml - type: "classification_match" diff --git a/plugins/coder-eval/reference/criteria.md b/plugins/coder-eval/reference/criteria.md index 526e1fbdd..3a15fb1fd 100644 --- a/plugins/coder-eval/reference/criteria.md +++ b/plugins/coder-eval/reference/criteria.md @@ -70,7 +70,7 @@ Match a single label written by the agent to a file against ground truth. | --- | --- | | `path` | Path to the file (relative to sandbox) containing the agent's predicted label; may be a glob matching exactly one file | | `expected_label` | Ground-truth label for this row | -| `allowed_labels` | Canonical label set. File content not in this set is treated as '(other)'. | +| `allowed_labels` | Canonical label set. File content not in this set, or a file that cannot be read, is treated as '(other)'. | Optional: diff --git a/src/coder_eval/criteria/classification_match.py b/src/coder_eval/criteria/classification_match.py index 52261a26a..23cc368af 100644 --- a/src/coder_eval/criteria/classification_match.py +++ b/src/coder_eval/criteria/classification_match.py @@ -50,11 +50,16 @@ def _check_impl( context: CheckContext | None = None, ) -> CriterionResult: expected = self._canonicalize(criterion.expected_label, criterion) + read_error: str | None = None try: raw = sandbox.get_file_content(criterion.path) except FileNotFoundError: observed = _SENTINEL_NONE + except (OSError, ValueError) as e: + # CE039: a miss that stays in the rollup; `error` keeps the cause countable. + observed = _SENTINEL_OTHER + read_error = f"{type(e).__name__}: {e}" else: content = raw.strip() observed = self._canonicalize(content, criterion) if content else _SENTINEL_NONE @@ -64,12 +69,15 @@ def _check_impl( score = 1.0 if observed == expected else 0.0 details = f"observed={observed!r}, expected={expected!r}" + if read_error is not None: + details += f" (unreadable: {read_error})" return ClassificationCriterionResult( criterion_type=criterion.type, description=criterion.description, score=score, details=details, + error=read_error, observed_label=observed, expected_label=expected, ) diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 1e661aa73..fbca08967 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -987,8 +987,9 @@ class ClassificationMatchCriterion(BaseSuccessCriterion): compares it to ``expected_label``. The observed label is the canonical form from ``allowed_labels`` when it matches; otherwise ``'(none)'`` when the file is missing / empty and ``'(other)'`` when the content is not in the - allowed set. Both are recorded as sentinels so the suite rollup can show - them as real failure classes in the confusion matrix. + allowed set or the file cannot be read. Both are recorded as sentinels + so the suite rollup can show them as real failure classes in the + confusion matrix. The checker returns a ``ClassificationCriterionResult`` (subclass of ``CriterionResult``) carrying observed + expected labels, which the @@ -1015,7 +1016,9 @@ class ClassificationMatchCriterion(BaseSuccessCriterion): expected_label: str = Field(description="Ground-truth label for this row") allowed_labels: list[str] = Field( min_length=1, - description="Canonical label set. File content not in this set is treated as '(other)'.", + description=( + "Canonical label set. File content not in this set, or a file that cannot be read, is treated as '(other)'." + ), ) case_sensitive: bool = Field( default=False, diff --git a/tests/test_classification_match.py b/tests/test_classification_match.py index f3efced48..94f657de9 100644 --- a/tests/test_classification_match.py +++ b/tests/test_classification_match.py @@ -10,6 +10,7 @@ import yaml from coder_eval.criteria.classification_match import ClassificationMatchChecker +from coder_eval.errors import CheckerMisuseError from coder_eval.models import ( AgentKind, ClassificationCriterionResult, @@ -17,22 +18,27 @@ CriterionResult, EvaluationResult, FinalStatus, + SandboxConfig, TaskDefinition, TaskResult, ) from coder_eval.reports.markdown import _compute_suite_rollup, _render_suite_markdown +from coder_eval.sandbox import Sandbox class _FakeSandbox: """Minimal sandbox surface needed by ClassificationMatchChecker.get_file_content.""" - def __init__(self, files: dict[str, str]): + def __init__(self, files: dict[str, str | Exception]): self._files = files def get_file_content(self, path: str) -> str: if path not in self._files: raise FileNotFoundError(path) - return self._files[path] + content = self._files[path] + if isinstance(content, Exception): + raise content + return content def _check( @@ -40,7 +46,7 @@ def _check( path: str, expected_label: str, allowed_labels: list[str], - files: dict[str, str], + files: dict[str, str | Exception], case_sensitive: bool = False, ) -> ClassificationCriterionResult: crit = ClassificationMatchCriterion( @@ -150,6 +156,65 @@ def test_empty_file_becomes_none_sentinel(self) -> None: assert result.score == 0.0 assert result.observed_label == "(none)" + @pytest.mark.parametrize( + "error", + [ + ValueError("Pattern 'out/*.txt' matches 2 files — refusing to guess which to grade"), + UnicodeDecodeError("utf-8", b"\xff\xfe", 0, 1, "invalid start byte"), + IsADirectoryError("result.txt"), + PermissionError("result.txt"), + ], + ) + def test_unreadable_file_becomes_other_sentinel(self, error: Exception) -> None: + result = _check( + path="result.txt", + expected_label="positive", + allowed_labels=["positive", "negative"], + files={"result.txt": error}, + ) + assert result.score == 0.0 + assert result.observed_label == "(other)" + assert result.error is not None + assert result.error.startswith(f"{type(error).__name__}: ") + + def test_checker_misuse_still_escalates(self) -> None: + crit = ClassificationMatchCriterion( + description="label match", + path="../outside.txt", + expected_label="positive", + allowed_labels=["positive", "negative"], + ) + sandbox = _FakeSandbox({"../outside.txt": CheckerMisuseError("path escapes the sandbox")}) + with pytest.raises(CheckerMisuseError): + ClassificationMatchChecker().check(crit, sandbox) # type: ignore[arg-type] + + @pytest.mark.parametrize("case", ["ambiguous_glob", "not_utf8", "directory"]) + def test_unreadable_file_in_a_real_sandbox(self, tmp_path: Path, case: str) -> None: + work = tmp_path / "work" + (work / "out").mkdir(parents=True) + if case == "ambiguous_glob": + (work / "out" / "a.txt").write_text("positive") + (work / "out" / "b.txt").write_text("positive") + elif case == "not_utf8": + (work / "out" / "a.txt").write_bytes(b"\xff\xfe") + else: + (work / "out" / "a.txt").mkdir() + sandbox = Sandbox(SandboxConfig(driver="tempdir"), task_id="t") + sandbox.sandbox_dir = work + crit = ClassificationMatchCriterion( + description="label match", + path="out/*.txt" if case == "ambiguous_glob" else "out/a.txt", + expected_label="negative", + allowed_labels=["positive", "negative"], + ) + + result = ClassificationMatchChecker().check(crit, sandbox) + + assert isinstance(result, ClassificationCriterionResult) + assert result.observed_label == "(other)" + assert result.score == 0.0 + assert result.error is not None + class TestClassificationMatchAggregate: """Across-row aggregate: accuracy / recall / F1 / confusion.""" @@ -352,6 +417,23 @@ def test_thresholds_one_fails(self, tmp_path: Path) -> None: assert by_metric["recall.negative"].passed is False assert by_metric["recall.negative"].actual_value == pytest.approx(0.5) + def test_unreadable_row_counts_as_a_miss(self, tmp_path: Path) -> None: + crit = self._criterion(suite_thresholds={"accuracy": 0.9}) + checker = ClassificationMatchChecker() + rows = self._mislabeled_rows() + unreadable = _FakeSandbox({"result.txt": ValueError("Pattern 'result.txt' matches 2 files")}) + r3_crit = crit.model_copy(update={"expected_label": "negative"}) + rows[2].result.success_criteria_results = [checker.check(r3_crit, unreadable)] # type: ignore[arg-type] + + rollup = _compute_suite_rollup("s", "v1", rows, tmp_path, task_criteria=[crit]) + agg = rollup.criterion_aggregates[0] + assert agg.metrics["accuracy"] == pytest.approx(2 / 3) + assert agg.details["total_pairs"] == 3 + assert {"expected": "negative", "observed": "(other)", "count": 1} in agg.details["confusion"] + assert agg.passed is False + stats = next(cs for cs in rollup.criterion_stats if cs.criterion_type == "classification_match") + assert stats.error_count == 1 + def test_threshold_on_missing_metric_fails(self, tmp_path: Path) -> None: rows = self._mislabeled_rows() crit = self._criterion(suite_thresholds={"does_not_exist": 0.5})