diff --git a/.claude/notes/lint-rules.md b/.claude/notes/lint-rules.md index 1044b58d9..5007746b3 100644 --- a/.claude/notes/lint-rules.md +++ b/.claude/notes/lint-rules.md @@ -961,3 +961,17 @@ the module as a parameter instead of a third copy. Taking the class name from the model follows CE056 importing `IN_CONTAINER_ENV` and CE057 deriving its target set from `SIDECAR_MODULES`. Alias resolution removes the *local binding* spelling, not every rename; taking the name from the model covers the rest. + +## CE071 + +The court-form verifier skipped checks when checkbox data or blank-field keys +were absent. Its shell runner treated pytest's successful exit as reward 1. +CE071 rejects skip and xfail calls or marks in every SkillsBench verifier whose +first line carries the `# Modified from benchflow-ai/skillsbench@` marker, so a +newly patched verifier is in scope without a list edit. The same test runs CE005 +on those files, because the removed defect was an `except Exception: pass` that +an inline `BaseException`-only check did not see. `dialogue-parser` is out of +scope: it is unpatched, and its runner scores passed / total, so a skip lowers +the reward instead of passing it. Real PDF regression tests, including two +whole-verifier pytest runs, anchor the court-form field names, extraction and +exit code to the shipped input. diff --git a/pyproject.toml b/pyproject.toml index 430073d58..7a89f5ab6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ Changelog = "https://github.com/UiPath/coder_eval/blob/main/CHANGELOG.md" [project.optional-dependencies] dev = [ + "pypdf[crypto]>=7.0.1", "pytest-asyncio>=1.3.0", "pytest>=9.0.2", "pytest-mock>=3.15.1", @@ -242,8 +243,8 @@ constraint-dependencies = [ [tool.ruff] line-length = 120 target-version = "py313" -# Vendored third-party sample tasks (benchflow-ai/skillsbench) are kept verbatim; -# they are not framework code and are not linted/formatted. +# Vendored third-party sample tasks (benchflow-ai/skillsbench) include local +# verifier patches documented in their README. Keep them outside framework formatting. extend-exclude = ["tasks/samples"] [tool.ruff.lint] @@ -311,6 +312,7 @@ external = [ "CE065", "CE066", "CE068", + "CE071", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/tasks/samples/skillsbench/README.md b/tasks/samples/skillsbench/README.md index 0584d5d7b..55acbc57c 100644 --- a/tasks/samples/skillsbench/README.md +++ b/tasks/samples/skillsbench/README.md @@ -7,18 +7,33 @@ evaluation (not just a smoke test) with one command. They are vendored from [**benchflow-ai/skillsbench**](https://github.com/benchflow-ai/skillsbench) (Apache-2.0), pinned at commit [`95b3e60`](https://github.com/benchflow-ai/skillsbench/tree/95b3e60de7a075090e2f9137f32e0d24da1a289b). -Each sample keeps the upstream prompt, inputs, and verifier; only the container -base and the coder_eval task wiring were added. The one local change is to the -`3d-scan-calc` verifier, which is patched to grade against fixed expected -values instead of re-deriving them from the agent-writable `scan_data.stl` -(issue #233). The other verifiers are unchanged and can still read -agent-writable inputs: `dialogue-parser` takes part of its ground truth from -`/app/script.txt`. - -That patch closes only the `scan_data.stl` vector. Grading still runs in the -agent's container, so the `pytest` on `PATH`, auto-loaded `pytest11` plugins, -site-packages, and the `$TASK_DIR` verifier copy remain tamperable by the -agent. See the follow-up issue +Each sample keeps the upstream prompt and inputs. The container base and +coder_eval task wiring were added locally, along with two verifier patches: + +- `3d-scan-calc` grades against fixed expected values instead of re-deriving + them from the agent-writable `scan_data.stl` (issue #233). +- `court-form-filling` fails when required checkbox data or fields that must + remain blank are missing (issue #234). Its blank-field checks also replace + a nonexistent county field with the omitted third trial court address field. + AcroForm read and field dereference errors propagate to the test runner. + The output must keep the SC-100 AcroForm fields: a flattened PDF scores 0, + even when it renders correctly. Checkbox grading reads AcroForm values only. + The verifier's XFA fallback cannot parse the shipped form's datasets, as + upstream, and now emits a warning when that happens. + Its environment pins pypdf 7.0.1 (upstream 5.1.0) and cryptography 50.0.2 + (upstream 43.0.3) to pick up their security fixes. It keeps upstream's + `PyPDF2==3.0.1`, the final release of that deprecated package, which has + known advisories with no fixed version. + +The `dialogue-parser` verifier is unchanged and takes part of its ground truth +from the agent-writable `/app/script.txt`. The `court-form-filling` verifier +also reads the agent-writable `/root/sc100-blank.pdf` for its input-versus-output +identity check. + +The `3d-scan-calc` patch closes only the `scan_data.stl` vector. Grading still +runs in the agent's container, so the `pytest` on `PATH`, auto-loaded `pytest11` +plugins, site-packages, and the `$TASK_DIR` verifier copy remain tamperable by +the agent. See the follow-up issue [#238](https://github.com/UiPath/coder_eval/issues/238). ## What they do @@ -56,8 +71,8 @@ coder-eval run tasks/samples/skillsbench/*/*.yaml Each task builds its container, runs the agent (Claude Sonnet 4.6 by default) in an isolated sandbox, then scores the result with the upstream verifier (for -`3d-scan-calc`, the locally patched one described above). Budget roughly 1–3 -minutes per task. +`3d-scan-calc` and `court-form-filling`, the local patches described above). +Budget roughly 1–3 minutes per task. ## Interpreting the results @@ -87,7 +102,7 @@ is non-deterministic, so exact numbers vary from run to run: | Task | Typical score | |------|---------------| | `3d-scan-calc` | 1.00 | -| `court-form-filling` | 1.00 | +| `court-form-filling` | TBD (re-baseline after #234) | | `dialogue-parser` | ~0.83 | ## How it works (brief) @@ -101,6 +116,6 @@ is non-deterministic, so exact numbers vary from run to run: These tasks are derived from benchflow-ai/skillsbench, licensed under Apache-2.0. The prompts, input data, and verifiers are the work of their original authors -(credited in each upstream task's `task.md`). The `3d-scan-calc` verifier is -patched locally to grade against fixed expected values. See the pinned commit -above for the upstream source of truth. +(credited in each upstream task's `task.md`). The `3d-scan-calc` and +`court-form-filling` verifiers have the local patches described above. See the +pinned commit above for the upstream source of truth. diff --git a/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml b/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml index 5a104ef56..0a09fd16e 100644 --- a/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml +++ b/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml @@ -46,9 +46,13 @@ run_limits: turn_timeout: 300 success_criteria: - # Run the skillsbench verifier verbatim: copy it from the task dir into a - # writable /verifier, run its test.sh (writes /logs/verifier/reward.txt), and - # emit that reward (0.0-1.0) as the criterion score. + # Run the skillsbench verifier (patched locally to fail on missing checkbox + # data or form fields, #234): copy it from the task dir into a writable + # /verifier, run its test.sh (writes /logs/verifier/reward.txt), and emit + # that reward (0.0-1.0) as the criterion score. + # Grading still runs in the agent's container, so the pytest on PATH, + # auto-loaded pytest11 plugins, site-packages and the $TASK_DIR verifier + # copy remain tamperable; see the follow-up issue #238. - type: run_command command: | set -e diff --git a/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile b/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile index b571aebc3..69c5d8aed 100644 --- a/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile +++ b/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile @@ -11,12 +11,12 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends poppler-utils \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir \ - pypdf==5.1.0 \ + pypdf==7.0.1 \ fillpdf==0.7.3 \ pdfrw==0.4 \ PyPDF2==3.0.1 \ reportlab==4.2.5 \ - cryptography==43.0.3 \ + cryptography==50.0.2 \ pytest==8.4.1 \ pytest-json-ctrf==0.3.5 diff --git a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py index e5c5b5281..2239e3e38 100644 --- a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py +++ b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py @@ -1,3 +1,4 @@ +# Modified from benchflow-ai/skillsbench@95b3e60: fail on missing form data (issue #234). """ Tests for court-form-filling task (SC-100 Small Claims Court form). @@ -11,6 +12,7 @@ import re import subprocess +import warnings import xml.etree.ElementTree as ET from pathlib import Path @@ -25,8 +27,10 @@ def extract_xfa_xml(pdf_path: Path) -> str: Extract XFA XML datasets from a PDF file. Returns the raw XML string or empty string if not found. """ + from pypdf import PdfReader + from pypdf.errors import PdfReadError + try: - from pypdf import PdfReader reader = PdfReader(str(pdf_path)) # Try to access XFA data @@ -42,8 +46,8 @@ def extract_xfa_xml(pdf_path: Path) -> str: if stream_name == "datasets": stream_obj = xfa[i + 1].get_object() return stream_obj.get_data().decode("utf-8", errors="ignore") - except Exception: - pass + except (KeyError, PdfReadError) as exc: + warnings.warn(f"XFA datasets could not be read; using AcroForm checkboxes only: {exc!r}", stacklevel=2) return "" @@ -63,26 +67,23 @@ def extract_checkbox_values(pdf_path: Path) -> dict: checkbox_values.update(parse_xfa_checkboxes(xfa_xml)) # Method 2: Try pypdf get_fields() for AcroForms - try: - from pypdf import PdfReader - reader = PdfReader(str(pdf_path)) - fields = reader.get_fields() or {} - - for name, field in fields.items(): - # Check if it's a checkbox/radio field - if "Checkbox" in name or field.get("/FT") in ["/Btn"]: - value = field.get("/V", "") - if value: - if hasattr(value, "get_object"): - value = str(value.get_object()) - else: - value = str(value) - # Normalize value - value = value.strip("/") - if value and value not in ["Off", "None", ""]: - checkbox_values[name] = value - except Exception: - pass + from pypdf import PdfReader + reader = PdfReader(str(pdf_path)) + fields = reader.get_fields() or {} + + for name, field in fields.items(): + # Check if it's a checkbox/radio field + if "Checkbox" in name or field.get("/FT") in ["/Btn"]: + value = field.get("/V", "") + if value: + if hasattr(value, "get_object"): + value = str(value.get_object()) + else: + value = str(value) + # Normalize value + value = value.strip("/") + if value and value not in ["Off", "None", ""]: + checkbox_values[name] = value return checkbox_values @@ -120,8 +121,8 @@ def find_checkboxes(element, path=""): find_checkboxes(child, current_path) find_checkboxes(root) - except Exception: - pass + except ET.ParseError as exc: + warnings.warn(f"XFA datasets did not parse; using AcroForm checkboxes only: {exc}", stacklevel=2) return checkbox_values @@ -268,7 +269,6 @@ def normalize_text(text: str) -> str: ("SC-100[0].Page1[0].CaptionRight[0].CN[0].CaseName[0]", "page1_case_name"), ("SC-100[0].Page1[0].CaptionRight[0].CN[0].CaseNumber[0]", "page1_case_number"), ("SC-100[0].Page1[0].CaptionRight[0].County[0].CourtInfo[0]", "page1_court_info"), - ("SC-100[0].Page1[0].CaptionRight[0].County[0].County[0]", "page1_county"), # Trial date section 1 ("SC-100[0].Page1[0].Order[0].List1[0].LI1[0].TrialDate1[0]", "page1_trial_date1"), ("SC-100[0].Page1[0].Order[0].List1[0].LI1[0].TrialTime1[0]", "page1_trial_time1"), @@ -283,6 +283,7 @@ def normalize_text(text: str) -> str: ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDate3[0]", "page1_trial_date3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialTIme3[0]", "page1_trial_time3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDepartment3[0]", "page1_trial_dept3"), + ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateCourtAdd3[0]", "page1_court_addr3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateClerkSign[0]", "page1_clerk_sign"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateClerkSignDate[0]", "page1_clerk_date"), # Second plaintiff/defendant (only one of each in this case) @@ -334,23 +335,25 @@ def checkbox_data(): @pytest.fixture(scope="module") def pdf_field_values(): """Extract all field values from the filled PDF using pypdf.""" - from pypdf import PdfReader - if not OUTPUT_FILE.exists(): pytest.fail(f"Output file not found at {OUTPUT_FILE}") - reader = PdfReader(str(OUTPUT_FILE)) + return extract_field_values(OUTPUT_FILE) + + +def extract_field_values(pdf_path: Path) -> dict[str, str]: + """Return every AcroForm field value as a string ("" when unset).""" + from pypdf import PdfReader + + reader = PdfReader(str(pdf_path)) fields = reader.get_fields() or {} - field_values = {} + field_values: dict[str, str] = {} for name, field in fields.items(): value = field.get("/V", "") if value: if hasattr(value, "get_object"): - try: - value = str(value.get_object()) - except: - value = str(value) + value = str(value.get_object()) else: value = str(value) field_values[name] = value if value else "" @@ -421,9 +424,6 @@ def test_checkbox_value(self, checkbox_data, field_pattern, description, expecte } if not matching_fields: - # If no checkboxes found at all, it might be extraction failure - if not checkbox_data: - pytest.skip("No checkbox data could be extracted from PDF (XFA extraction may not be supported)") pytest.fail( f"Checkbox '{description}' (pattern: {field_pattern}) not found. " f"Available checkboxes: {list(checkbox_data.keys())}" @@ -459,9 +459,6 @@ def test_checkbox_unchecked(self, checkbox_data, field_pattern, description): } if not matching_fields: - # If no checkboxes found at all, it might be extraction failure - if not checkbox_data: - pytest.skip("No checkbox data could be extracted from PDF") # Field not found means it's likely unchecked (not in filled data) return @@ -479,6 +476,7 @@ class TestEmptyFields: ids=[f[1] for f in EMPTY_FIELDS]) def test_field_empty(self, pdf_field_values, field_name, description): """Verify field is empty or has minimal placeholder.""" - value = pdf_field_values.get(field_name, "") + assert field_name in pdf_field_values, f"Field '{description}' not found in PDF form" + value = pdf_field_values[field_name] is_empty = not value or value in ["", "None"] or len(value) < 3 assert is_empty, f"Field '{description}' should be empty, got: '{value}'" diff --git a/tests/lint/rules/ce071_patched_verifier_no_skip.py b/tests/lint/rules/ce071_patched_verifier_no_skip.py new file mode 100644 index 000000000..ad6eb51a3 --- /dev/null +++ b/tests/lint/rules/ce071_patched_verifier_no_skip.py @@ -0,0 +1,61 @@ +"""CE071: A locally patched binary-reward verifier must not skip or xfail. + +Flags ``pytest.skip`` / ``pytest.importorskip`` / ``pytest.xfail`` calls and +``pytest.mark.skip`` / ``skipif`` / ``xfail`` marks. A binary verifier's runner +writes reward 1 when pytest exits 0, so a skipped check counts as a pass. + +Scope: every ``tasks/samples/skillsbench/*/verifier/test_outputs.py`` whose first +line starts with ``PATCHED_MARKER``. The test runs this rule together with CE005, +which catches the broad ``except Exception: pass`` swallow. ``dialogue-parser`` is +out of scope because it is unpatched and its runner scores passed / total, so a +skip lowers its reward instead of passing it. + +BLIND SPOT: aliased imports (``from pytest import skip``) and dynamically built +calls are not resolved. + +Add ``# noqa: CE071`` on the offending line for a deliberate exception. + +Rationale: .claude/notes/lint-rules.md § CE071 +""" + +import ast + +from tests.lint.rules.base import BaseRule + + +PATCHED_MARKER = "# Modified from benchflow-ai/skillsbench@" +_OUTCOME_CALLS = frozenset({"skip", "importorskip", "xfail"}) +_SKIP_MARKS = frozenset({"skip", "skipif", "xfail"}) + + +def _is_pytest_name(node: ast.expr) -> bool: + return isinstance(node, ast.Name) and node.id == "pytest" + + +def _is_pytest_outcome_call(node: ast.Call) -> bool: + func = node.func + return isinstance(func, ast.Attribute) and func.attr in _OUTCOME_CALLS and _is_pytest_name(func.value) + + +def _is_pytest_skip_mark(node: ast.Attribute) -> bool: + owner = node.value + return ( + node.attr in _SKIP_MARKS + and isinstance(owner, ast.Attribute) + and owner.attr == "mark" + and _is_pytest_name(owner.value) + ) + + +class PatchedVerifierNoSkip(BaseRule): + id = "CE071" + + def visit_Call(self, node: ast.Call) -> None: + if _is_pytest_outcome_call(node): + self.violation(node, "patched binary verifier must fail, not skip or xfail") + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> None: + if _is_pytest_skip_mark(node): + self.violation(node, "patched binary verifier must not carry a skip or xfail mark") + self.generic_visit(node) diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index c166dfd5a..bb1a7289e 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -5399,3 +5399,64 @@ def test_shares_manifest_skill_dirs_with_the_runtime_allowlist(self): from coder_eval.isolation import eval_material assert eval_material.manifest_skill_dirs is manifest_skill_dirs + + +@pytest.mark.lint +class TestCE071PatchedBinaryVerifiersDoNotSkip: + """CE071 + CE005 over every SkillsBench verifier that carries the patch marker. + + The rule and its scope are documented in ``tests/lint/rules/ce071_patched_verifier_no_skip.py``. + """ + + @staticmethod + def _patched_verifiers() -> list[Path]: + from tests.lint.rules.ce071_patched_verifier_no_skip import PATCHED_MARKER + + root = SRC.parent / "tasks" / "samples" / "skillsbench" + return sorted( + path + for path in root.glob("*/verifier/test_outputs.py") + if path.read_text(encoding="utf-8").startswith(PATCHED_MARKER) + ) + + @staticmethod + def _run(tmp_path: Path, source: str) -> list[str]: + from tests.lint.rules.ce071_patched_verifier_no_skip import PatchedVerifierNoSkip + from tests.lint.rules.no_silent_except import NoSilentExcept + + path = tmp_path / "test_outputs.py" + path.write_text(source, encoding="utf-8") + return [v.rule_id for v in check_paths([path], rules=[PatchedVerifierNoSkip, NoSilentExcept])] + + def test_patched_verifiers_are_found(self) -> None: + names = {path.parts[-3] for path in self._patched_verifiers()} + assert {"3d-scan-calc", "court-form-filling"} <= names + + def test_patched_verifiers_clean(self) -> None: + from tests.lint.rules.ce071_patched_verifier_no_skip import PatchedVerifierNoSkip + from tests.lint.rules.no_silent_except import NoSilentExcept + + violations = check_paths(self._patched_verifiers(), rules=[PatchedVerifierNoSkip, NoSilentExcept]) + assert not violations, "\n".join(str(v) for v in violations) + + @pytest.mark.parametrize( + ("source", "rule_id"), + [ + ("pytest.skip('missing')", "CE071"), + ("pytest.importorskip('pypdf')", "CE071"), + ("pytest.xfail('missing')", "CE071"), + ("@pytest.mark.skip\ndef test(): pass", "CE071"), + ("@pytest.mark.skipif(True, reason='missing')\ndef test(): pass", "CE071"), + ("@pytest.mark.xfail\ndef test(): pass", "CE071"), + ("try: read()\nexcept: pass", "CE005"), + ("try: read()\nexcept Exception: pass", "CE005"), + ], + ) + def test_detects_silent_outcomes(self, tmp_path: Path, source: str, rule_id: str) -> None: + assert self._run(tmp_path, source) == [rule_id] + + def test_allows_failure_and_narrow_exception(self, tmp_path: Path) -> None: + assert not self._run(tmp_path, "try: read()\nexcept ValueError: pytest.fail('invalid')") + + def test_honours_noqa(self, tmp_path: Path) -> None: + assert not self._run(tmp_path, "pytest.skip('optional') # noqa: CE071") diff --git a/tests/test_skillsbench_court_form_verifier.py b/tests/test_skillsbench_court_form_verifier.py new file mode 100644 index 000000000..0205df99e --- /dev/null +++ b/tests/test_skillsbench_court_form_verifier.py @@ -0,0 +1,212 @@ +"""Regression tests for the vendored court-form-filling verifier.""" + +from __future__ import annotations + +import importlib.util +import re +import subprocess +import sys +import tomllib +import xml.etree.ElementTree as ET +from pathlib import Path +from unittest.mock import Mock + +import pytest +from pypdf import PdfReader, PdfWriter +from pypdf.errors import PdfReadError + + +_REPO = Path(__file__).resolve().parents[1] +_TASK_DIR = _REPO / "tasks" / "samples" / "skillsbench" / "court-form-filling" +_BLANK_PDF = _TASK_DIR / "environment" / "sc100-blank.pdf" +_SPEC = importlib.util.spec_from_file_location("court_form_verifier", _TASK_DIR / "verifier" / "test_outputs.py") +assert _SPEC is not None and _SPEC.loader is not None +_VERIFIER = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_VERIFIER) + +pytestmark = pytest.mark.filterwarnings("ignore:XFA datasets:UserWarning") + +type RequiredCase = tuple[str, str, str, bool] +type NamedCase = tuple[str, str] + + +def _write_correct_fill(output: Path) -> Path: + reader = PdfReader(_BLANK_PDF) + fields = reader.get_fields() + assert fields is not None + values = { + name: expected + for pattern, _, expected, _ in _VERIFIER.REQUIRED_CHECKBOXES + for name, field in fields.items() + if pattern in name and expected in field.get("/_States_", []) + } + writer = PdfWriter(clone_from=reader) + writer.update_page_form_field_values(list(writer.pages), values, auto_regenerate=False) + writer.write(output) + return output + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_missing_fails(case: RequiredCase) -> None: + with pytest.raises(pytest.fail.Exception, match="not found"): + _VERIFIER.TestCheckboxes().test_checkbox_value({}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_wrong_value_fails(case: RequiredCase) -> None: + with pytest.raises(AssertionError, match="expected value"): + _VERIFIER.TestCheckboxes().test_checkbox_value({case[0]: "wrong"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_correct_value_passes(case: RequiredCase) -> None: + _VERIFIER.TestCheckboxes().test_checkbox_value({case[0]: case[2]}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.UNCHECKED_CHECKBOXES, ids=lambda case: case[1]) +def test_unchecked_checkbox_absent_passes(case: NamedCase) -> None: + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked({}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.UNCHECKED_CHECKBOXES, ids=lambda case: case[1]) +def test_unchecked_checkbox_checked_fails(case: NamedCase) -> None: + with pytest.raises(AssertionError, match="should be unchecked"): + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked({case[0]: "/1"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_missing_fails(case: NamedCase) -> None: + values = {name: "" for name, _ in _VERIFIER.EMPTY_FIELDS if name != case[0]} + with pytest.raises(AssertionError, match="not found"): + _VERIFIER.TestEmptyFields().test_field_empty(values, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_filled_fails(case: NamedCase) -> None: + with pytest.raises(AssertionError, match="should be empty"): + _VERIFIER.TestEmptyFields().test_field_empty({case[0]: "filled"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_blank_passes(case: NamedCase) -> None: + _VERIFIER.TestEmptyFields().test_field_empty({case[0]: ""}, *case) + + +def test_expected_fields_exist_in_shipped_pdf() -> None: + fields = PdfReader(_BLANK_PDF).get_fields() + assert fields is not None + for name, _ in _VERIFIER.EMPTY_FIELDS: + assert name in fields, name + assert fields[name]["/FT"] == "/Tx", name + for pattern, *_ in [*_VERIFIER.REQUIRED_CHECKBOXES, *_VERIFIER.UNCHECKED_CHECKBOXES]: + matches = [field for name, field in fields.items() if pattern in name] + assert matches, pattern + assert all(field["/FT"] == "/Btn" for field in matches), pattern + + +def test_correct_pdf_field_roundtrip_passes(tmp_path: Path) -> None: + output = _write_correct_fill(tmp_path / "filled.pdf") + checkboxes = _VERIFIER.extract_checkbox_values(output) + field_values = _VERIFIER.extract_field_values(output) + for case in _VERIFIER.REQUIRED_CHECKBOXES: + _VERIFIER.TestCheckboxes().test_checkbox_value(checkboxes, *case) + for case in _VERIFIER.UNCHECKED_CHECKBOXES: + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked(checkboxes, *case) + for case in _VERIFIER.EMPTY_FIELDS: + _VERIFIER.TestEmptyFields().test_field_empty(field_values, *case) + + +def test_blank_pdf_required_checkboxes_fail() -> None: + checkboxes = _VERIFIER.extract_checkbox_values(_BLANK_PDF) + for case in _VERIFIER.REQUIRED_CHECKBOXES: + with pytest.raises(pytest.fail.Exception, match="not found"): + _VERIFIER.TestCheckboxes().test_checkbox_value(checkboxes, *case) + + +def test_pdf_without_form_fields_fails(tmp_path: Path) -> None: + output = tmp_path / "no-form.pdf" + writer = PdfWriter() + writer.add_blank_page(width=612, height=792) + writer.write(output) + values = _VERIFIER.extract_field_values(output) + assert values == {} + for case in _VERIFIER.EMPTY_FIELDS: + with pytest.raises(AssertionError, match="not found"): + _VERIFIER.TestEmptyFields().test_field_empty(values, *case) + + +def test_checkbox_extraction_preserves_pdf_error(tmp_path: Path) -> None: + output = tmp_path / "invalid.pdf" + output.write_bytes(b"not a PDF") + with pytest.raises(PdfReadError): + _VERIFIER.extract_checkbox_values(output) + + +def test_field_extraction_preserves_dereference_error(monkeypatch: pytest.MonkeyPatch) -> None: + value = Mock() + value.get_object.side_effect = RuntimeError("broken field reference") + monkeypatch.setattr("pypdf.PdfReader", Mock(return_value=Mock(get_fields=lambda: {"field": {"/V": value}}))) + with pytest.raises(RuntimeError, match="broken field reference"): + _VERIFIER.extract_field_values(_BLANK_PDF) + + +def test_shipped_xfa_datasets_do_not_parse_and_warn() -> None: + """Pins the upstream contract: checkbox grading uses AcroForm values only. + + The namespace cleanup leaves ``xfa:`` attribute prefixes, so the shipped datasets never parse. Making them + parse would grade XFA values too (a blank ``"0"`` matches unchecked patterns such as ``Checkbox11``), which + is a separate grading change. + """ + xfa_xml = _VERIFIER.extract_xfa_xml(_BLANK_PDF) + assert xfa_xml + with pytest.warns(UserWarning, match="XFA datasets did not parse.*unbound prefix"): + assert _VERIFIER.parse_xfa_checkboxes(xfa_xml) == {} + + +def _run_verifier(tmp_path: Path, output_pdf: Path) -> tuple[int, ET.Element]: + """Run the verifier as test.sh does, with OUTPUT_FILE pointed at ``output_pdf``.""" + source = (_TASK_DIR / "verifier" / "test_outputs.py").read_text(encoding="utf-8") + original = 'OUTPUT_FILE = Path("/root/sc100-filled.pdf")' + assert source.count(original) == 1 + verifier = tmp_path / "test_outputs.py" + verifier.write_text(source.replace(original, f"OUTPUT_FILE = Path({str(output_pdf)!r})"), encoding="utf-8") + (tmp_path / "pytest.ini").write_text("[pytest]\n", encoding="utf-8") + junit = tmp_path / "junit.xml" + command = [sys.executable, "-m", "pytest", str(verifier), "-k", "Checkboxes or EmptyFields"] + proc = subprocess.run( + [*command, "-p", "no:cacheprovider", "-c", str(tmp_path / "pytest.ini"), f"--junitxml={junit}"], + cwd=tmp_path, + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + suite = ET.parse(junit).getroot().find("testsuite") + assert suite is not None, proc.stdout + return proc.returncode, suite + + +def test_verifier_run_fails_blank_form_without_skips(tmp_path: Path) -> None: + returncode, suite = _run_verifier(tmp_path, _BLANK_PDF) + assert returncode != 0 + assert int(suite.get("failures", "0")) > 0 + assert suite.get("skipped") == "0" + + +def test_verifier_run_passes_correct_fill(tmp_path: Path) -> None: + output = _write_correct_fill(tmp_path / "filled.pdf") + returncode, suite = _run_verifier(tmp_path, output) + assert returncode == 0 + assert int(suite.get("tests", "0")) > 0 + assert suite.get("skipped") == "0" + + +def test_dockerfile_pins_match_lock() -> None: + """The regression tests above only vouch for the PDF stack the verifier image installs.""" + dockerfile = (_TASK_DIR / "environment" / "Dockerfile").read_text(encoding="utf-8") + locked = { + package["name"]: package["version"] + for package in tomllib.loads((_REPO / "uv.lock").read_text(encoding="utf-8"))["package"] + } + for name in ("pypdf", "cryptography"): + assert re.findall(rf"\b{name}==(\S+)", dockerfile) == [locked[name]], name diff --git a/uv.lock b/uv.lock index ffb3a0738..34898ad2e 100644 --- a/uv.lock +++ b/uv.lock @@ -596,6 +596,7 @@ dev = [ { name = "mcp" }, { name = "pip-audit" }, { name = "pre-commit" }, + { name = "pypdf", extra = ["crypto"] }, { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -636,6 +637,7 @@ requires-dist = [ { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.5.1" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, + { name = "pypdf", extras = ["crypto"], marker = "extra == 'dev'", specifier = ">=7.0.1" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.408" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" }, @@ -2406,6 +2408,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pypdf" +version = "7.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/c2/4c34b3229abae3e2fcc7809e6cf80fdea159771a37142f475fbf1009c809/pypdf-7.0.1.tar.gz", hash = "sha256:0a78bb99efb41e9088260e9a8e5da134a5cbb9a322014744177bf29b5267ea6d", size = 7206748, upload-time = "2026-11-10T11:47:22.582Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/a9/ae11deef21eaac5113f8c3848da1c42423b7abeca0abc349bfec7abf4c4e/pypdf-7.0.1-py3-none-any.whl", hash = "sha256:6131b764b046282357037309e7ef2a1b7679d4a5c53eed3db2bfbf7c673ddc2b", size = 416136, upload-time = "2026-11-10T11:47:20.716Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyright" version = "1.1.408"