Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/TASK_DEFINITION_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -1349,7 +1349,7 @@ The judge runs with the evaluator's API credentials and can execute arbitrary Ba

Matches a single label the agent wrote to a file against ground truth — the file-based classifier. Reads the file, normalizes the content (strip, and lowercase unless `case_sensitive`), and compares it to `expected_label`. **Binary scoring:** `1.0` on a match, else `0.0`.

The observed label is the canonical form from `allowed_labels` when the content matches; otherwise `(none)` when the file is missing/empty and `(other)` when the content isn't in the allowed set. Both sentinels are recorded so a suite rollup shows them as real failure classes in the confusion matrix.
The observed label is the canonical form from `allowed_labels` when the content matches; otherwise `(none)` when the file is missing/empty and `(other)` when the content isn't in the allowed set or the file can't be read (for example, it isn't UTF-8, or a glob matches more than one file). Both sentinels are recorded so a suite rollup shows them as real failure classes in the confusion matrix.

```yaml
- type: "classification_match"
Expand Down
2 changes: 1 addition & 1 deletion plugins/coder-eval/reference/criteria.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ Match a single label written by the agent to a file against ground truth.
| --- | --- |
| `path` | Path to the file (relative to sandbox) containing the agent's predicted label; may be a glob matching exactly one file |
| `expected_label` | Ground-truth label for this row |
| `allowed_labels` | Canonical label set. File content not in this set is treated as '(other)'. |
| `allowed_labels` | Canonical label set. File content not in this set, or a file that cannot be read, is treated as '(other)'. |

Optional:

Expand Down
8 changes: 8 additions & 0 deletions src/coder_eval/criteria/classification_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,16 @@ def _check_impl(
context: CheckContext | None = None,
) -> CriterionResult:
expected = self._canonicalize(criterion.expected_label, criterion)
read_error: str | None = None

try:
raw = sandbox.get_file_content(criterion.path)
except FileNotFoundError:
observed = _SENTINEL_NONE
except (OSError, ValueError) as e:
# CE039: a miss that stays in the rollup; `error` keeps the cause countable.
observed = _SENTINEL_OTHER
read_error = f"{type(e).__name__}: {e}"
else:
content = raw.strip()
observed = self._canonicalize(content, criterion) if content else _SENTINEL_NONE
Expand All @@ -64,12 +69,15 @@ def _check_impl(

score = 1.0 if observed == expected else 0.0
details = f"observed={observed!r}, expected={expected!r}"
if read_error is not None:
details += f" (unreadable: {read_error})"

return ClassificationCriterionResult(
criterion_type=criterion.type,
description=criterion.description,
score=score,
details=details,
error=read_error,
observed_label=observed,
expected_label=expected,
)
Expand Down
9 changes: 6 additions & 3 deletions src/coder_eval/models/criteria.py
Original file line number Diff line number Diff line change
Expand Up @@ -987,8 +987,9 @@ class ClassificationMatchCriterion(BaseSuccessCriterion):
compares it to ``expected_label``. The observed label is the canonical form
from ``allowed_labels`` when it matches; otherwise ``'(none)'`` when the
file is missing / empty and ``'(other)'`` when the content is not in the
allowed set. Both are recorded as sentinels so the suite rollup can show
them as real failure classes in the confusion matrix.
allowed set or the file cannot be read. Both are recorded as sentinels
so the suite rollup can show them as real failure classes in the
confusion matrix.

The checker returns a ``ClassificationCriterionResult`` (subclass of
``CriterionResult``) carrying observed + expected labels, which the
Expand All @@ -1015,7 +1016,9 @@ class ClassificationMatchCriterion(BaseSuccessCriterion):
expected_label: str = Field(description="Ground-truth label for this row")
allowed_labels: list[str] = Field(
min_length=1,
description="Canonical label set. File content not in this set is treated as '(other)'.",
description=(
"Canonical label set. File content not in this set, or a file that cannot be read, is treated as '(other)'."
),
)
case_sensitive: bool = Field(
default=False,
Expand Down
88 changes: 85 additions & 3 deletions tests/test_classification_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,37 +10,43 @@
import yaml

from coder_eval.criteria.classification_match import ClassificationMatchChecker
from coder_eval.errors import CheckerMisuseError
from coder_eval.models import (
AgentKind,
ClassificationCriterionResult,
ClassificationMatchCriterion,
CriterionResult,
EvaluationResult,
FinalStatus,
SandboxConfig,
TaskDefinition,
TaskResult,
)
from coder_eval.reports.markdown import _compute_suite_rollup, _render_suite_markdown
from coder_eval.sandbox import Sandbox


class _FakeSandbox:
"""Minimal sandbox surface needed by ClassificationMatchChecker.get_file_content."""

def __init__(self, files: dict[str, str]):
def __init__(self, files: dict[str, str | Exception]):
self._files = files

def get_file_content(self, path: str) -> str:
if path not in self._files:
raise FileNotFoundError(path)
return self._files[path]
content = self._files[path]
if isinstance(content, Exception):
raise content
return content


def _check(
*,
path: str,
expected_label: str,
allowed_labels: list[str],
files: dict[str, str],
files: dict[str, str | Exception],
case_sensitive: bool = False,
) -> ClassificationCriterionResult:
crit = ClassificationMatchCriterion(
Expand Down Expand Up @@ -150,6 +156,65 @@ def test_empty_file_becomes_none_sentinel(self) -> None:
assert result.score == 0.0
assert result.observed_label == "(none)"

@pytest.mark.parametrize(
"error",
[
ValueError("Pattern 'out/*.txt' matches 2 files — refusing to guess which to grade"),
UnicodeDecodeError("utf-8", b"\xff\xfe", 0, 1, "invalid start byte"),
IsADirectoryError("result.txt"),
PermissionError("result.txt"),
],
)
def test_unreadable_file_becomes_other_sentinel(self, error: Exception) -> None:
result = _check(
path="result.txt",
expected_label="positive",
allowed_labels=["positive", "negative"],
files={"result.txt": error},
)
assert result.score == 0.0
assert result.observed_label == "(other)"
assert result.error is not None
assert result.error.startswith(f"{type(error).__name__}: ")

def test_checker_misuse_still_escalates(self) -> None:
crit = ClassificationMatchCriterion(
description="label match",
path="../outside.txt",
expected_label="positive",
allowed_labels=["positive", "negative"],
)
sandbox = _FakeSandbox({"../outside.txt": CheckerMisuseError("path escapes the sandbox")})
with pytest.raises(CheckerMisuseError):
ClassificationMatchChecker().check(crit, sandbox) # type: ignore[arg-type]

@pytest.mark.parametrize("case", ["ambiguous_glob", "not_utf8", "directory"])
def test_unreadable_file_in_a_real_sandbox(self, tmp_path: Path, case: str) -> None:
work = tmp_path / "work"
(work / "out").mkdir(parents=True)
if case == "ambiguous_glob":
(work / "out" / "a.txt").write_text("positive")
(work / "out" / "b.txt").write_text("positive")
elif case == "not_utf8":
(work / "out" / "a.txt").write_bytes(b"\xff\xfe")
else:
(work / "out" / "a.txt").mkdir()
sandbox = Sandbox(SandboxConfig(driver="tempdir"), task_id="t")
sandbox.sandbox_dir = work
crit = ClassificationMatchCriterion(
description="label match",
path="out/*.txt" if case == "ambiguous_glob" else "out/a.txt",
expected_label="negative",
allowed_labels=["positive", "negative"],
)

result = ClassificationMatchChecker().check(crit, sandbox)

assert isinstance(result, ClassificationCriterionResult)
assert result.observed_label == "(other)"
assert result.score == 0.0
assert result.error is not None


class TestClassificationMatchAggregate:
"""Across-row aggregate: accuracy / recall / F1 / confusion."""
Expand Down Expand Up @@ -352,6 +417,23 @@ def test_thresholds_one_fails(self, tmp_path: Path) -> None:
assert by_metric["recall.negative"].passed is False
assert by_metric["recall.negative"].actual_value == pytest.approx(0.5)

def test_unreadable_row_counts_as_a_miss(self, tmp_path: Path) -> None:
crit = self._criterion(suite_thresholds={"accuracy": 0.9})
checker = ClassificationMatchChecker()
rows = self._mislabeled_rows()
unreadable = _FakeSandbox({"result.txt": ValueError("Pattern 'result.txt' matches 2 files")})
r3_crit = crit.model_copy(update={"expected_label": "negative"})
rows[2].result.success_criteria_results = [checker.check(r3_crit, unreadable)] # type: ignore[arg-type]

rollup = _compute_suite_rollup("s", "v1", rows, tmp_path, task_criteria=[crit])
agg = rollup.criterion_aggregates[0]
assert agg.metrics["accuracy"] == pytest.approx(2 / 3)
assert agg.details["total_pairs"] == 3
assert {"expected": "negative", "observed": "(other)", "count": 1} in agg.details["confusion"]
assert agg.passed is False
stats = next(cs for cs in rollup.criterion_stats if cs.criterion_type == "classification_match")
assert stats.error_count == 1

def test_threshold_on_missing_metric_fails(self, tmp_path: Path) -> None:
rows = self._mislabeled_rows()
crit = self._criterion(suite_thresholds={"does_not_exist": 0.5})
Expand Down
Loading