From 6c6c6f201d144669f55ba63d37f50d787e9ec7a7 Mon Sep 17 00:00:00 2001 From: gezerui0124 Date: Sat, 31 Oct 2026 03:22:54 +0800 Subject: [PATCH 1/4] fix(skillsbench): reject incomplete court forms --- tasks/samples/skillsbench/README.md | 30 +++-- .../verifier/test_outputs.py | 11 +- tests/test_skillsbench_court_form_verifier.py | 115 ++++++++++++++++++ 3 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 tests/test_skillsbench_court_form_verifier.py diff --git a/tasks/samples/skillsbench/README.md b/tasks/samples/skillsbench/README.md index 0584d5d7b..defccd361 100644 --- a/tasks/samples/skillsbench/README.md +++ b/tasks/samples/skillsbench/README.md @@ -7,15 +7,19 @@ evaluation (not just a smoke test) with one command. They are vendored from [**benchflow-ai/skillsbench**](https://github.com/benchflow-ai/skillsbench) (Apache-2.0), pinned at commit [`95b3e60`](https://github.com/benchflow-ai/skillsbench/tree/95b3e60de7a075090e2f9137f32e0d24da1a289b). -Each sample keeps the upstream prompt, inputs, and verifier; only the container -base and the coder_eval task wiring were added. The one local change is to the -`3d-scan-calc` verifier, which is patched to grade against fixed expected -values instead of re-deriving them from the agent-writable `scan_data.stl` -(issue #233). The other verifiers are unchanged and can still read -agent-writable inputs: `dialogue-parser` takes part of its ground truth from -`/app/script.txt`. - -That patch closes only the `scan_data.stl` vector. Grading still runs in the +Each sample keeps the upstream prompt and inputs. The container base and +coder_eval task wiring were added locally, along with two verifier patches: + +- `3d-scan-calc` grades against fixed expected values instead of re-deriving + them from the agent-writable `scan_data.stl` (issue #233). +- `court-form-filling` fails when required checkbox data or fields that must + remain blank are missing (issue #234). Its blank-field checks also replace + a nonexistent county field with the omitted third trial court address field. + +The `dialogue-parser` verifier is unchanged and takes part of its ground truth +from the agent-writable `/app/script.txt`. + +The `3d-scan-calc` patch closes only the `scan_data.stl` vector. Grading still runs in the agent's container, so the `pytest` on `PATH`, auto-loaded `pytest11` plugins, site-packages, and the `$TASK_DIR` verifier copy remain tamperable by the agent. See the follow-up issue @@ -56,7 +60,7 @@ coder-eval run tasks/samples/skillsbench/*/*.yaml Each task builds its container, runs the agent (Claude Sonnet 4.6 by default) in an isolated sandbox, then scores the result with the upstream verifier (for -`3d-scan-calc`, the locally patched one described above). Budget roughly 1–3 +`3d-scan-calc` and `court-form-filling`, the local patches described above). Budget roughly 1–3 minutes per task. ## Interpreting the results @@ -101,6 +105,6 @@ is non-deterministic, so exact numbers vary from run to run: These tasks are derived from benchflow-ai/skillsbench, licensed under Apache-2.0. The prompts, input data, and verifiers are the work of their original authors -(credited in each upstream task's `task.md`). The `3d-scan-calc` verifier is -patched locally to grade against fixed expected values. See the pinned commit -above for the upstream source of truth. +(credited in each upstream task's `task.md`). The `3d-scan-calc` and +`court-form-filling` verifiers have the local patches described above. See the +pinned commit above for the upstream source of truth. diff --git a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py index e5c5b5281..44bf75b5e 100644 --- a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py +++ b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py @@ -268,7 +268,6 @@ def normalize_text(text: str) -> str: ("SC-100[0].Page1[0].CaptionRight[0].CN[0].CaseName[0]", "page1_case_name"), ("SC-100[0].Page1[0].CaptionRight[0].CN[0].CaseNumber[0]", "page1_case_number"), ("SC-100[0].Page1[0].CaptionRight[0].County[0].CourtInfo[0]", "page1_court_info"), - ("SC-100[0].Page1[0].CaptionRight[0].County[0].County[0]", "page1_county"), # Trial date section 1 ("SC-100[0].Page1[0].Order[0].List1[0].LI1[0].TrialDate1[0]", "page1_trial_date1"), ("SC-100[0].Page1[0].Order[0].List1[0].LI1[0].TrialTime1[0]", "page1_trial_time1"), @@ -283,6 +282,7 @@ def normalize_text(text: str) -> str: ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDate3[0]", "page1_trial_date3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialTIme3[0]", "page1_trial_time3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDepartment3[0]", "page1_trial_dept3"), + ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateCourtAdd3[0]", "page1_court_addr3"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateClerkSign[0]", "page1_clerk_sign"), ("SC-100[0].Page1[0].Order[0].List1[0].LI3[0].TrialDateClerkSignDate[0]", "page1_clerk_date"), # Second plaintiff/defendant (only one of each in this case) @@ -421,9 +421,6 @@ def test_checkbox_value(self, checkbox_data, field_pattern, description, expecte } if not matching_fields: - # If no checkboxes found at all, it might be extraction failure - if not checkbox_data: - pytest.skip("No checkbox data could be extracted from PDF (XFA extraction may not be supported)") pytest.fail( f"Checkbox '{description}' (pattern: {field_pattern}) not found. " f"Available checkboxes: {list(checkbox_data.keys())}" @@ -459,9 +456,6 @@ def test_checkbox_unchecked(self, checkbox_data, field_pattern, description): } if not matching_fields: - # If no checkboxes found at all, it might be extraction failure - if not checkbox_data: - pytest.skip("No checkbox data could be extracted from PDF") # Field not found means it's likely unchecked (not in filled data) return @@ -479,6 +473,7 @@ class TestEmptyFields: ids=[f[1] for f in EMPTY_FIELDS]) def test_field_empty(self, pdf_field_values, field_name, description): """Verify field is empty or has minimal placeholder.""" - value = pdf_field_values.get(field_name, "") + assert field_name in pdf_field_values, f"Field '{description}' not found in PDF form" + value = pdf_field_values[field_name] is_empty = not value or value in ["", "None"] or len(value) < 3 assert is_empty, f"Field '{description}' should be empty, got: '{value}'" diff --git a/tests/test_skillsbench_court_form_verifier.py b/tests/test_skillsbench_court_form_verifier.py new file mode 100644 index 000000000..465e117a5 --- /dev/null +++ b/tests/test_skillsbench_court_form_verifier.py @@ -0,0 +1,115 @@ +"""Regression tests for the vendored court-form-filling verifier.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +_VERIFIER = ( + Path(__file__).resolve().parents[1] + / "tasks" + / "samples" + / "skillsbench" + / "court-form-filling" + / "verifier" + / "test_outputs.py" +) + + +@pytest.mark.parametrize( + ("check_class", "fixture_name", "values", "exit_code", "summary"), + [ + ("TestCheckboxes", "checkbox_data", "{}", 1, "6 failed"), + ("TestUncheckedCheckboxes", "checkbox_data", "{}", 0, "6 passed"), + ("TestEmptyFields", "pdf_field_values", "{}", 1, "20 failed"), + ( + "TestEmptyFields", + "pdf_field_values", + '{name: "" for name, _ in EMPTY_FIELDS[1:]}', + 1, + "1 failed, 19 passed", + ), + ( + "TestEmptyFields", + "pdf_field_values", + '{name: "filled" for name, _ in EMPTY_FIELDS}', + 1, + "20 failed", + ), + ( + "TestEmptyFields", + "pdf_field_values", + '{name: "" for name, _ in EMPTY_FIELDS}', + 0, + "20 passed", + ), + ( + "TestCheckboxes", + "checkbox_data", + "{pattern: expected for pattern, _, expected, _ in REQUIRED_CHECKBOXES}", + 0, + "6 passed", + ), + ( + "TestCheckboxes", + "checkbox_data", + '{pattern: "wrong" for pattern, _, _, _ in REQUIRED_CHECKBOXES}', + 1, + "6 failed", + ), + ( + "TestUncheckedCheckboxes", + "checkbox_data", + '{pattern: "/1" for pattern, _ in UNCHECKED_CHECKBOXES}', + 1, + "6 failed", + ), + ], + ids=[ + "missing-required-boxes", + "unchecked-boxes", + "missing-fields", + "one-missing-field", + "nonblank-fields", + "blank-fields", + "correct-boxes", + "wrong-boxes", + "unexpected-checked-boxes", + ], +) +def test_verifier_outcome( + tmp_path: Path, check_class: str, fixture_name: str, values: str, exit_code: int, summary: str +) -> None: + verifier = tmp_path / "test_outputs.py" + verifier.write_text( + _VERIFIER.read_text() + f"\n\n@pytest.fixture\ndef {fixture_name}():\n return {values}\n", + encoding="utf-8", + ) + config = tmp_path / "pytest.ini" + config.write_text("[pytest]\n", encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-c", + str(config), + f"{verifier}::{check_class}", + "-q", + "-p", + "no:cacheprovider", + ], + cwd=tmp_path, + env={**os.environ, "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == exit_code, result.stdout + result.stderr + assert summary in result.stdout, result.stdout + assert "skipped" not in result.stdout, result.stdout From 2ec6190b06f99b11740c656bc7b41b43ab55a2af Mon Sep 17 00:00:00 2001 From: gezerui0124 Date: Sat, 31 Oct 2026 09:25:59 +0800 Subject: [PATCH 2/4] fix(skillsbench): verify real forms and preserve AcroForm errors --- .claude/notes/lint-rules.md | 9 + pyproject.toml | 6 +- tasks/samples/skillsbench/README.md | 24 +- .../court-form-filling/environment/Dockerfile | 2 +- .../verifier/test_outputs.py | 43 ++- tests/test_custom_lint.py | 69 +++++ tests/test_skillsbench_court_form_verifier.py | 246 ++++++++++-------- uv.lock | 16 ++ 8 files changed, 274 insertions(+), 141 deletions(-) diff --git a/.claude/notes/lint-rules.md b/.claude/notes/lint-rules.md index 1044b58d9..32a6f2fa3 100644 --- a/.claude/notes/lint-rules.md +++ b/.claude/notes/lint-rules.md @@ -961,3 +961,12 @@ the module as a parameter instead of a third copy. Taking the class name from the model follows CE056 importing `IN_CONTAINER_ENV` and CE057 deriving its target set from `SIDECAR_MODULES`. Alias resolution removes the *local binding* spelling, not every rename; taking the name from the model covers the rest. + +## CE071 — patched binary verifiers must not skip + +The court-form verifier skipped checks when checkbox data or blank-field keys +were absent. Its shell runner treated pytest's successful exit as reward 1. +The guard rejects skip and xfail calls or marks in the two locally patched +binary-reward SkillsBench verifiers. It also rejects bare `except:` and +`except BaseException:` handlers. Real PDF regression tests +also anchor the court-form field names and extraction to the shipped input. diff --git a/pyproject.toml b/pyproject.toml index 2cb30ce71..d7628741a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ Changelog = "https://github.com/UiPath/coder_eval/blob/main/CHANGELOG.md" [project.optional-dependencies] dev = [ + "pypdf[crypto]==6.21.4", "pytest-asyncio>=1.3.0", "pytest>=9.0.2", "pytest-mock>=3.15.1", @@ -241,8 +242,8 @@ constraint-dependencies = [ [tool.ruff] line-length = 120 target-version = "py313" -# Vendored third-party sample tasks (benchflow-ai/skillsbench) are kept verbatim; -# they are not framework code and are not linted/formatted. +# Vendored third-party sample tasks (benchflow-ai/skillsbench) include local +# verifier patches documented in their README. Keep them outside framework formatting. extend-exclude = ["tasks/samples"] [tool.ruff.lint] @@ -310,6 +311,7 @@ external = [ "CE065", "CE066", "CE068", + "CE071", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/tasks/samples/skillsbench/README.md b/tasks/samples/skillsbench/README.md index defccd361..bb07d7fdb 100644 --- a/tasks/samples/skillsbench/README.md +++ b/tasks/samples/skillsbench/README.md @@ -15,14 +15,18 @@ coder_eval task wiring were added locally, along with two verifier patches: - `court-form-filling` fails when required checkbox data or fields that must remain blank are missing (issue #234). Its blank-field checks also replace a nonexistent county field with the omitted third trial court address field. + AcroForm read and field dereference errors propagate to the test runner. + Its environment pins pypdf 6.21.4 (upstream 5.1.0) to remove known advisories. The `dialogue-parser` verifier is unchanged and takes part of its ground truth -from the agent-writable `/app/script.txt`. - -The `3d-scan-calc` patch closes only the `scan_data.stl` vector. Grading still runs in the -agent's container, so the `pytest` on `PATH`, auto-loaded `pytest11` plugins, -site-packages, and the `$TASK_DIR` verifier copy remain tamperable by the -agent. See the follow-up issue +from the agent-writable `/app/script.txt`. The `court-form-filling` verifier +also reads the agent-writable `/root/sc100-blank.pdf` for its input-versus-output +identity check. + +The `3d-scan-calc` patch closes only the `scan_data.stl` vector. Grading still +runs in the agent's container, so the `pytest` on `PATH`, auto-loaded `pytest11` +plugins, site-packages, and the `$TASK_DIR` verifier copy remain tamperable by +the agent. See the follow-up issue [#238](https://github.com/UiPath/coder_eval/issues/238). ## What they do @@ -60,8 +64,8 @@ coder-eval run tasks/samples/skillsbench/*/*.yaml Each task builds its container, runs the agent (Claude Sonnet 4.6 by default) in an isolated sandbox, then scores the result with the upstream verifier (for -`3d-scan-calc` and `court-form-filling`, the local patches described above). Budget roughly 1–3 -minutes per task. +`3d-scan-calc` and `court-form-filling`, the local patches described above). +Budget roughly 1–3 minutes per task. ## Interpreting the results @@ -94,6 +98,10 @@ is non-deterministic, so exact numbers vary from run to run: | `court-form-filling` | 1.00 | | `dialogue-parser` | ~0.83 | +The court-form score above predates the verifier fix for issue #234 and the +pypdf update. Those historical scores are not comparable with scores from +the updated sample. + ## How it works (brief) - **`environment/Dockerfile`** rebases the upstream task onto `coder-eval-agent` and bakes the inputs at the paths the prompt expects. diff --git a/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile b/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile index b571aebc3..54c85bda2 100644 --- a/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile +++ b/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends poppler-utils \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir \ - pypdf==5.1.0 \ + pypdf==6.21.4 \ fillpdf==0.7.3 \ pdfrw==0.4 \ PyPDF2==3.0.1 \ diff --git a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py index 44bf75b5e..83b81f3e6 100644 --- a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py +++ b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py @@ -1,3 +1,4 @@ +# Modified from benchflow-ai/skillsbench@95b3e60: require form fields and preserve AcroForm read and field dereference errors (issue #234). """ Tests for court-form-filling task (SC-100 Small Claims Court form). @@ -63,26 +64,23 @@ def extract_checkbox_values(pdf_path: Path) -> dict: checkbox_values.update(parse_xfa_checkboxes(xfa_xml)) # Method 2: Try pypdf get_fields() for AcroForms - try: - from pypdf import PdfReader - reader = PdfReader(str(pdf_path)) - fields = reader.get_fields() or {} - - for name, field in fields.items(): - # Check if it's a checkbox/radio field - if "Checkbox" in name or field.get("/FT") in ["/Btn"]: - value = field.get("/V", "") - if value: - if hasattr(value, "get_object"): - value = str(value.get_object()) - else: - value = str(value) - # Normalize value - value = value.strip("/") - if value and value not in ["Off", "None", ""]: - checkbox_values[name] = value - except Exception: - pass + from pypdf import PdfReader + reader = PdfReader(str(pdf_path)) + fields = reader.get_fields() or {} + + for name, field in fields.items(): + # Check if it's a checkbox/radio field + if "Checkbox" in name or field.get("/FT") in ["/Btn"]: + value = field.get("/V", "") + if value: + if hasattr(value, "get_object"): + value = str(value.get_object()) + else: + value = str(value) + # Normalize value + value = value.strip("/") + if value and value not in ["Off", "None", ""]: + checkbox_values[name] = value return checkbox_values @@ -347,10 +345,7 @@ def pdf_field_values(): value = field.get("/V", "") if value: if hasattr(value, "get_object"): - try: - value = str(value.get_object()) - except: - value = str(value) + value = str(value.get_object()) else: value = str(value) field_values[name] = value if value else "" diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index c166dfd5a..7eb1e52df 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -5399,3 +5399,72 @@ def test_shares_manifest_skill_dirs_with_the_runtime_allowlist(self): from coder_eval.isolation import eval_material assert eval_material.manifest_skill_dirs is manifest_skill_dirs + + +@pytest.mark.lint +class TestCE071PatchedBinaryVerifiersDoNotSkip: + """CE071: patched binary verifiers must not skip, xfail, or catch BaseException. + + Scope: the locally patched 3d-scan-calc and court-form-filling verifiers. + Blind spots: imported aliases and dynamically constructed calls are not resolved. + """ + + @staticmethod + def _offenders(source: str) -> list[int]: + import ast + + return [ + node.lineno + for node in ast.walk(ast.parse(source)) + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "pytest" + and node.func.attr in {"skip", "importorskip", "xfail"} + ) + or ( + isinstance(node, ast.Attribute) + and node.attr in {"skip", "skipif", "xfail"} + and isinstance(node.value, ast.Attribute) + and node.value.attr == "mark" + and isinstance(node.value.value, ast.Name) + and node.value.value.id == "pytest" + ) + or ( + isinstance(node, ast.ExceptHandler) + and (node.type is None or (isinstance(node.type, ast.Name) and node.type.id == "BaseException")) + ) + ] + + @pytest.mark.parametrize("slug", ["3d-scan-calc", "court-form-filling"]) + def test_patched_verifier(self, slug: str) -> None: + path = ( + Path(__file__).resolve().parents[1] + / "tasks" + / "samples" + / "skillsbench" + / slug + / "verifier/test_outputs.py" + ) + offenders = self._offenders(path.read_text(encoding="utf-8")) + assert not offenders, f"CE071: silent outcome or broad catch in {path} at lines {offenders}" + + @pytest.mark.parametrize( + "source", + [ + "pytest.skip('missing')", + "pytest.importorskip('pypdf')", + "pytest.xfail('missing')", + "@pytest.mark.skip\ndef test(): pass", + "@pytest.mark.skipif(True, reason='missing')\ndef test(): pass", + "@pytest.mark.xfail\ndef test(): pass", + "try: read()\nexcept: pass", + "try: read()\nexcept BaseException: pass", + ], + ) + def test_detects_silent_outcomes(self, source: str) -> None: + assert self._offenders(source) + + def test_allows_failure_and_narrow_exception(self) -> None: + assert not self._offenders("try: read()\nexcept ValueError: pytest.fail('invalid')") diff --git a/tests/test_skillsbench_court_form_verifier.py b/tests/test_skillsbench_court_form_verifier.py index 465e117a5..fb3910b72 100644 --- a/tests/test_skillsbench_court_form_verifier.py +++ b/tests/test_skillsbench_court_form_verifier.py @@ -2,114 +2,148 @@ from __future__ import annotations -import os -import subprocess -import sys +import importlib.util from pathlib import Path +from unittest.mock import Mock import pytest +from pypdf import PdfReader, PdfWriter +from pypdf.errors import PdfReadError -_VERIFIER = ( - Path(__file__).resolve().parents[1] - / "tasks" - / "samples" - / "skillsbench" - / "court-form-filling" - / "verifier" - / "test_outputs.py" -) - - -@pytest.mark.parametrize( - ("check_class", "fixture_name", "values", "exit_code", "summary"), - [ - ("TestCheckboxes", "checkbox_data", "{}", 1, "6 failed"), - ("TestUncheckedCheckboxes", "checkbox_data", "{}", 0, "6 passed"), - ("TestEmptyFields", "pdf_field_values", "{}", 1, "20 failed"), - ( - "TestEmptyFields", - "pdf_field_values", - '{name: "" for name, _ in EMPTY_FIELDS[1:]}', - 1, - "1 failed, 19 passed", - ), - ( - "TestEmptyFields", - "pdf_field_values", - '{name: "filled" for name, _ in EMPTY_FIELDS}', - 1, - "20 failed", - ), - ( - "TestEmptyFields", - "pdf_field_values", - '{name: "" for name, _ in EMPTY_FIELDS}', - 0, - "20 passed", - ), - ( - "TestCheckboxes", - "checkbox_data", - "{pattern: expected for pattern, _, expected, _ in REQUIRED_CHECKBOXES}", - 0, - "6 passed", - ), - ( - "TestCheckboxes", - "checkbox_data", - '{pattern: "wrong" for pattern, _, _, _ in REQUIRED_CHECKBOXES}', - 1, - "6 failed", - ), - ( - "TestUncheckedCheckboxes", - "checkbox_data", - '{pattern: "/1" for pattern, _ in UNCHECKED_CHECKBOXES}', - 1, - "6 failed", - ), - ], - ids=[ - "missing-required-boxes", - "unchecked-boxes", - "missing-fields", - "one-missing-field", - "nonblank-fields", - "blank-fields", - "correct-boxes", - "wrong-boxes", - "unexpected-checked-boxes", - ], -) -def test_verifier_outcome( - tmp_path: Path, check_class: str, fixture_name: str, values: str, exit_code: int, summary: str -) -> None: - verifier = tmp_path / "test_outputs.py" - verifier.write_text( - _VERIFIER.read_text() + f"\n\n@pytest.fixture\ndef {fixture_name}():\n return {values}\n", - encoding="utf-8", - ) - config = tmp_path / "pytest.ini" - config.write_text("[pytest]\n", encoding="utf-8") - result = subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "-c", - str(config), - f"{verifier}::{check_class}", - "-q", - "-p", - "no:cacheprovider", - ], - cwd=tmp_path, - env={**os.environ, "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, - capture_output=True, - text=True, - timeout=30, - ) - assert result.returncode == exit_code, result.stdout + result.stderr - assert summary in result.stdout, result.stdout - assert "skipped" not in result.stdout, result.stdout +_TASK_DIR = Path(__file__).resolve().parents[1] / "tasks" / "samples" / "skillsbench" / "court-form-filling" +_BLANK_PDF = _TASK_DIR / "environment" / "sc100-blank.pdf" +_SPEC = importlib.util.spec_from_file_location("court_form_verifier", _TASK_DIR / "verifier" / "test_outputs.py") +assert _SPEC is not None and _SPEC.loader is not None +_VERIFIER = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_VERIFIER) + + +@pytest.fixture(autouse=True) +def reject_verifier_skips(monkeypatch: pytest.MonkeyPatch) -> None: + """A direct verifier call must fail if it tries to skip a check.""" + monkeypatch.setattr(pytest, "skip", pytest.fail) + + +def test_verifier_skip_fails() -> None: + with pytest.raises(pytest.fail.Exception, match="missing"): + pytest.skip("missing") + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_missing_fails(case: tuple) -> None: + with pytest.raises(pytest.fail.Exception, match="not found"): + _VERIFIER.TestCheckboxes().test_checkbox_value({}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_wrong_value_fails(case: tuple) -> None: + with pytest.raises(AssertionError, match="expected value"): + _VERIFIER.TestCheckboxes().test_checkbox_value({case[0]: "wrong"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) +def test_required_checkbox_correct_value_passes(case: tuple) -> None: + _VERIFIER.TestCheckboxes().test_checkbox_value({case[0]: case[2]}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.UNCHECKED_CHECKBOXES, ids=lambda case: case[1]) +def test_unchecked_checkbox_absent_passes(case: tuple) -> None: + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked({}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.UNCHECKED_CHECKBOXES, ids=lambda case: case[1]) +def test_unchecked_checkbox_checked_fails(case: tuple) -> None: + with pytest.raises(AssertionError, match="should be unchecked"): + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked({case[0]: "/1"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_missing_fails(case: tuple) -> None: + values = {name: "" for name, _ in _VERIFIER.EMPTY_FIELDS if name != case[0]} + with pytest.raises(AssertionError, match="not found"): + _VERIFIER.TestEmptyFields().test_field_empty(values, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_filled_fails(case: tuple) -> None: + with pytest.raises(AssertionError, match="should be empty"): + _VERIFIER.TestEmptyFields().test_field_empty({case[0]: "filled"}, *case) + + +@pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) +def test_empty_field_blank_passes(case: tuple) -> None: + _VERIFIER.TestEmptyFields().test_field_empty({case[0]: ""}, *case) + + +def test_expected_fields_exist_in_shipped_pdf() -> None: + fields = PdfReader(_BLANK_PDF).get_fields() + assert fields is not None + for name, _ in _VERIFIER.EMPTY_FIELDS: + assert name in fields, name + assert fields[name]["/FT"] == "/Tx", name + for pattern, *_ in [*_VERIFIER.REQUIRED_CHECKBOXES, *_VERIFIER.UNCHECKED_CHECKBOXES]: + matches = [field for name, field in fields.items() if pattern in name] + assert matches, pattern + assert all(field["/FT"] == "/Btn" for field in matches), pattern + + +def test_correct_pdf_field_roundtrip_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + reader = PdfReader(_BLANK_PDF) + fields = reader.get_fields() + assert fields is not None + values = { + name: expected + for pattern, _, expected, _ in _VERIFIER.REQUIRED_CHECKBOXES + for name, field in fields.items() + if pattern in name and expected in field.get("/_States_", []) + } + output = tmp_path / "filled.pdf" + writer = PdfWriter(clone_from=reader) + writer.update_page_form_field_values(list(writer.pages), values, auto_regenerate=False) + writer.write(output) + monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", output) + checkboxes = _VERIFIER.checkbox_data.__wrapped__() + field_values = _VERIFIER.pdf_field_values.__wrapped__() + for case in _VERIFIER.REQUIRED_CHECKBOXES: + _VERIFIER.TestCheckboxes().test_checkbox_value(checkboxes, *case) + for case in _VERIFIER.UNCHECKED_CHECKBOXES: + _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked(checkboxes, *case) + for case in _VERIFIER.EMPTY_FIELDS: + _VERIFIER.TestEmptyFields().test_field_empty(field_values, *case) + + +def test_blank_pdf_required_checkboxes_fail() -> None: + checkboxes = _VERIFIER.extract_checkbox_values(_BLANK_PDF) + for case in _VERIFIER.REQUIRED_CHECKBOXES: + with pytest.raises(pytest.fail.Exception, match="not found"): + _VERIFIER.TestCheckboxes().test_checkbox_value(checkboxes, *case) + + +def test_pdf_without_form_fields_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + output = tmp_path / "no-form.pdf" + writer = PdfWriter() + writer.add_blank_page(width=612, height=792) + writer.write(output) + monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", output) + values = _VERIFIER.pdf_field_values.__wrapped__() + assert values == {} + for case in _VERIFIER.EMPTY_FIELDS: + with pytest.raises(AssertionError, match="not found"): + _VERIFIER.TestEmptyFields().test_field_empty(values, *case) + + +def test_checkbox_extraction_preserves_pdf_error(tmp_path: Path) -> None: + output = tmp_path / "invalid.pdf" + output.write_bytes(b"not a PDF") + with pytest.raises(PdfReadError): + _VERIFIER.extract_checkbox_values(output) + + +def test_field_extraction_preserves_dereference_error(monkeypatch: pytest.MonkeyPatch) -> None: + value = Mock() + value.get_object.side_effect = RuntimeError("broken field reference") + monkeypatch.setattr("pypdf.PdfReader", Mock(return_value=Mock(get_fields=lambda: {"field": {"/V": value}}))) + monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", _BLANK_PDF) + with pytest.raises(RuntimeError, match="broken field reference"): + _VERIFIER.pdf_field_values.__wrapped__() diff --git a/uv.lock b/uv.lock index 3dfa40491..39eec8516 100644 --- a/uv.lock +++ b/uv.lock @@ -596,6 +596,7 @@ dev = [ { name = "mcp" }, { name = "pip-audit" }, { name = "pre-commit" }, + { name = "pypdf", extra = ["crypto"] }, { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -636,6 +637,7 @@ requires-dist = [ { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.5.1" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, + { name = "pypdf", extras = ["crypto"], marker = "extra == 'dev'", specifier = "==6.21.4" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.408" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" }, @@ -2406,6 +2408,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pypdf" +version = "6.21.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyright" version = "1.1.408" From 4c62954fe377523c1af4956b6cbd06668015f494 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 13 Nov 2026 14:36:17 -0800 Subject: [PATCH 3/4] fix(skillsbench): pin pypdf 7.0.1 and harden court-form verifier tests pypdf 6.21.4 has five GHSA advisories fixed only in 7.0.0/7.0.1 (GHSA-cmpx-464x-82c3 XFA memory use, GHSA-rfv5-695j-j993, GHSA-fhff-j4m8-439h, GHSA-59jv-x5x2-gg4c, GHSA-mwj3-fpmc-hjcv). Bump the dev and Dockerfile pins to 7.0.1, and the image's cryptography pin from 43.0.3 to 50.0.2. - Add extract_field_values to the verifier so the tests call plain functions instead of the pytest-internal fixture __wrapped__. - Add a test that keeps the Dockerfile and dev pypdf pins equal. - README: state that PyPDF2 3.0.1 stays and has unfixed advisories. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- tasks/samples/skillsbench/README.md | 5 ++- .../court-form-filling/environment/Dockerfile | 4 +-- .../verifier/test_outputs.py | 13 +++++--- tests/test_skillsbench_court_form_verifier.py | 32 +++++++++++++------ uv.lock | 10 +++--- 6 files changed, 43 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d80883b10..402fa4850 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ Changelog = "https://github.com/UiPath/coder_eval/blob/main/CHANGELOG.md" [project.optional-dependencies] dev = [ - "pypdf[crypto]==6.21.4", + "pypdf[crypto]==7.0.1", "pytest-asyncio>=1.3.0", "pytest>=9.0.2", "pytest-mock>=3.15.1", diff --git a/tasks/samples/skillsbench/README.md b/tasks/samples/skillsbench/README.md index bb07d7fdb..8a932a56b 100644 --- a/tasks/samples/skillsbench/README.md +++ b/tasks/samples/skillsbench/README.md @@ -16,7 +16,10 @@ coder_eval task wiring were added locally, along with two verifier patches: remain blank are missing (issue #234). Its blank-field checks also replace a nonexistent county field with the omitted third trial court address field. AcroForm read and field dereference errors propagate to the test runner. - Its environment pins pypdf 6.21.4 (upstream 5.1.0) to remove known advisories. + Its environment pins pypdf 7.0.1 (upstream 5.1.0) and cryptography 50.0.2 + (upstream 43.0.3) to pick up their security fixes. It keeps upstream's + `PyPDF2==3.0.1`, the final release of that deprecated package, which has + known advisories with no fixed version. The `dialogue-parser` verifier is unchanged and takes part of its ground truth from the agent-writable `/app/script.txt`. The `court-form-filling` verifier diff --git a/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile b/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile index 54c85bda2..69c5d8aed 100644 --- a/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile +++ b/tasks/samples/skillsbench/court-form-filling/environment/Dockerfile @@ -11,12 +11,12 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends poppler-utils \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir \ - pypdf==6.21.4 \ + pypdf==7.0.1 \ fillpdf==0.7.3 \ pdfrw==0.4 \ PyPDF2==3.0.1 \ reportlab==4.2.5 \ - cryptography==43.0.3 \ + cryptography==50.0.2 \ pytest==8.4.1 \ pytest-json-ctrf==0.3.5 diff --git a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py index 83b81f3e6..9b226c893 100644 --- a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py +++ b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py @@ -1,4 +1,4 @@ -# Modified from benchflow-ai/skillsbench@95b3e60: require form fields and preserve AcroForm read and field dereference errors (issue #234). +# Modified from benchflow-ai/skillsbench@95b3e60: require form fields, preserve AcroForm read and field dereference errors, and expose extract_field_values (issue #234). """ Tests for court-form-filling task (SC-100 Small Claims Court form). @@ -332,12 +332,17 @@ def checkbox_data(): @pytest.fixture(scope="module") def pdf_field_values(): """Extract all field values from the filled PDF using pypdf.""" - from pypdf import PdfReader - if not OUTPUT_FILE.exists(): pytest.fail(f"Output file not found at {OUTPUT_FILE}") - reader = PdfReader(str(OUTPUT_FILE)) + return extract_field_values(OUTPUT_FILE) + + +def extract_field_values(pdf_path: Path) -> dict: + """Return every AcroForm field value as a string ("" when unset).""" + from pypdf import PdfReader + + reader = PdfReader(str(pdf_path)) fields = reader.get_fields() or {} field_values = {} diff --git a/tests/test_skillsbench_court_form_verifier.py b/tests/test_skillsbench_court_form_verifier.py index fb3910b72..ab884963c 100644 --- a/tests/test_skillsbench_court_form_verifier.py +++ b/tests/test_skillsbench_court_form_verifier.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util +import re +import tomllib from pathlib import Path from unittest.mock import Mock @@ -11,7 +13,8 @@ from pypdf.errors import PdfReadError -_TASK_DIR = Path(__file__).resolve().parents[1] / "tasks" / "samples" / "skillsbench" / "court-form-filling" +_REPO = Path(__file__).resolve().parents[1] +_TASK_DIR = _REPO / "tasks" / "samples" / "skillsbench" / "court-form-filling" _BLANK_PDF = _TASK_DIR / "environment" / "sc100-blank.pdf" _SPEC = importlib.util.spec_from_file_location("court_form_verifier", _TASK_DIR / "verifier" / "test_outputs.py") assert _SPEC is not None and _SPEC.loader is not None @@ -88,7 +91,7 @@ def test_expected_fields_exist_in_shipped_pdf() -> None: assert all(field["/FT"] == "/Btn" for field in matches), pattern -def test_correct_pdf_field_roundtrip_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_correct_pdf_field_roundtrip_passes(tmp_path: Path) -> None: reader = PdfReader(_BLANK_PDF) fields = reader.get_fields() assert fields is not None @@ -102,9 +105,8 @@ def test_correct_pdf_field_roundtrip_passes(tmp_path: Path, monkeypatch: pytest. writer = PdfWriter(clone_from=reader) writer.update_page_form_field_values(list(writer.pages), values, auto_regenerate=False) writer.write(output) - monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", output) - checkboxes = _VERIFIER.checkbox_data.__wrapped__() - field_values = _VERIFIER.pdf_field_values.__wrapped__() + checkboxes = _VERIFIER.extract_checkbox_values(output) + field_values = _VERIFIER.extract_field_values(output) for case in _VERIFIER.REQUIRED_CHECKBOXES: _VERIFIER.TestCheckboxes().test_checkbox_value(checkboxes, *case) for case in _VERIFIER.UNCHECKED_CHECKBOXES: @@ -120,13 +122,12 @@ def test_blank_pdf_required_checkboxes_fail() -> None: _VERIFIER.TestCheckboxes().test_checkbox_value(checkboxes, *case) -def test_pdf_without_form_fields_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_pdf_without_form_fields_fails(tmp_path: Path) -> None: output = tmp_path / "no-form.pdf" writer = PdfWriter() writer.add_blank_page(width=612, height=792) writer.write(output) - monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", output) - values = _VERIFIER.pdf_field_values.__wrapped__() + values = _VERIFIER.extract_field_values(output) assert values == {} for case in _VERIFIER.EMPTY_FIELDS: with pytest.raises(AssertionError, match="not found"): @@ -144,6 +145,17 @@ def test_field_extraction_preserves_dereference_error(monkeypatch: pytest.Monkey value = Mock() value.get_object.side_effect = RuntimeError("broken field reference") monkeypatch.setattr("pypdf.PdfReader", Mock(return_value=Mock(get_fields=lambda: {"field": {"/V": value}}))) - monkeypatch.setattr(_VERIFIER, "OUTPUT_FILE", _BLANK_PDF) with pytest.raises(RuntimeError, match="broken field reference"): - _VERIFIER.pdf_field_values.__wrapped__() + _VERIFIER.extract_field_values(_BLANK_PDF) + + +def test_dockerfile_pypdf_pin_matches_dev_pin() -> None: + """The regression tests above only vouch for the pypdf the verifier image installs.""" + dev = tomllib.loads((_REPO / "pyproject.toml").read_text(encoding="utf-8"))["project"]["optional-dependencies"][ + "dev" + ] + dev_pins = [re.fullmatch(r"pypdf(?:\[[^]]*\])?==(\S+)", dep) for dep in dev] + dockerfile_pins = re.findall( + r"\bpypdf==(\S+)", (_TASK_DIR / "environment" / "Dockerfile").read_text(encoding="utf-8") + ) + assert [m.group(1) for m in dev_pins if m] == dockerfile_pins diff --git a/uv.lock b/uv.lock index eba931a7c..d93d843d9 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.13" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -637,7 +637,7 @@ requires-dist = [ { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.5.1" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, - { name = "pypdf", extras = ["crypto"], marker = "extra == 'dev'", specifier = "==6.21.4" }, + { name = "pypdf", extras = ["crypto"], marker = "extra == 'dev'", specifier = "==7.0.1" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.408" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" }, @@ -2410,11 +2410,11 @@ wheels = [ [[package]] name = "pypdf" -version = "6.21.4" +version = "7.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/c2/4c34b3229abae3e2fcc7809e6cf80fdea159771a37142f475fbf1009c809/pypdf-7.0.1.tar.gz", hash = "sha256:0a78bb99efb41e9088260e9a8e5da134a5cbb9a322014744177bf29b5267ea6d", size = 7206748, upload-time = "2026-11-10T11:47:22.582Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" }, + { url = "https://files.pythonhosted.org/packages/d5/a9/ae11deef21eaac5113f8c3848da1c42423b7abeca0abc349bfec7abf4c4e/pypdf-7.0.1-py3-none-any.whl", hash = "sha256:6131b764b046282357037309e7ef2a1b7679d4a5c53eed3db2bfbf7c673ddc2b", size = 416136, upload-time = "2026-11-10T11:47:20.716Z" }, ] [package.optional-dependencies] From 4502f3ad8f96c193a2b51224fe6967921f4d32c9 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Mon, 16 Nov 2026 17:46:16 -0800 Subject: [PATCH 4/4] fix(skillsbench): address court-form review findings - Relax the dev pin to pypdf[crypto]>=7.0.1 and check the Dockerfile pypdf and cryptography pins against uv.lock. Regenerate uv.lock with a current uv so revision stays at 5. - Replace the verifier's silent XFA handlers with narrow handlers that warn. Keep upstream's AcroForm-only checkbox grading and pin it with a test, because parsing the XFA datasets would change grading. - Move CE071 to a BaseRule, take its scope from the "Modified from" marker, and run it with CE005 over the patched verifiers. - Add whole-verifier pytest runs for a blank form and a correct fill, and remove the redundant skip-to-fail fixture. - Document the AcroForm output contract, mark the court-form score TBD, and cite #234 and #238 in the task YAML. Co-Authored-By: Claude Opus 5.5 --- .claude/notes/lint-rules.md | 15 ++- pyproject.toml | 2 +- tasks/samples/skillsbench/README.md | 10 +- .../court-form-filling.yaml | 10 +- .../verifier/test_outputs.py | 19 +-- .../rules/ce071_patched_verifier_no_skip.py | 61 +++++++++ tests/test_custom_lint.py | 98 +++++++------- tests/test_skillsbench_court_form_verifier.py | 125 ++++++++++++------ uv.lock | 4 +- 9 files changed, 230 insertions(+), 114 deletions(-) create mode 100644 tests/lint/rules/ce071_patched_verifier_no_skip.py diff --git a/.claude/notes/lint-rules.md b/.claude/notes/lint-rules.md index 32a6f2fa3..5007746b3 100644 --- a/.claude/notes/lint-rules.md +++ b/.claude/notes/lint-rules.md @@ -962,11 +962,16 @@ Taking the class name from the model follows CE056 importing `IN_CONTAINER_ENV` deriving its target set from `SIDECAR_MODULES`. Alias resolution removes the *local binding* spelling, not every rename; taking the name from the model covers the rest. -## CE071 — patched binary verifiers must not skip +## CE071 The court-form verifier skipped checks when checkbox data or blank-field keys were absent. Its shell runner treated pytest's successful exit as reward 1. -The guard rejects skip and xfail calls or marks in the two locally patched -binary-reward SkillsBench verifiers. It also rejects bare `except:` and -`except BaseException:` handlers. Real PDF regression tests -also anchor the court-form field names and extraction to the shipped input. +CE071 rejects skip and xfail calls or marks in every SkillsBench verifier whose +first line carries the `# Modified from benchflow-ai/skillsbench@` marker, so a +newly patched verifier is in scope without a list edit. The same test runs CE005 +on those files, because the removed defect was an `except Exception: pass` that +an inline `BaseException`-only check did not see. `dialogue-parser` is out of +scope: it is unpatched, and its runner scores passed / total, so a skip lowers +the reward instead of passing it. Real PDF regression tests, including two +whole-verifier pytest runs, anchor the court-form field names, extraction and +exit code to the shipped input. diff --git a/pyproject.toml b/pyproject.toml index 402fa4850..7a89f5ab6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ Changelog = "https://github.com/UiPath/coder_eval/blob/main/CHANGELOG.md" [project.optional-dependencies] dev = [ - "pypdf[crypto]==7.0.1", + "pypdf[crypto]>=7.0.1", "pytest-asyncio>=1.3.0", "pytest>=9.0.2", "pytest-mock>=3.15.1", diff --git a/tasks/samples/skillsbench/README.md b/tasks/samples/skillsbench/README.md index 8a932a56b..55acbc57c 100644 --- a/tasks/samples/skillsbench/README.md +++ b/tasks/samples/skillsbench/README.md @@ -16,6 +16,10 @@ coder_eval task wiring were added locally, along with two verifier patches: remain blank are missing (issue #234). Its blank-field checks also replace a nonexistent county field with the omitted third trial court address field. AcroForm read and field dereference errors propagate to the test runner. + The output must keep the SC-100 AcroForm fields: a flattened PDF scores 0, + even when it renders correctly. Checkbox grading reads AcroForm values only. + The verifier's XFA fallback cannot parse the shipped form's datasets, as + upstream, and now emits a warning when that happens. Its environment pins pypdf 7.0.1 (upstream 5.1.0) and cryptography 50.0.2 (upstream 43.0.3) to pick up their security fixes. It keeps upstream's `PyPDF2==3.0.1`, the final release of that deprecated package, which has @@ -98,13 +102,9 @@ is non-deterministic, so exact numbers vary from run to run: | Task | Typical score | |------|---------------| | `3d-scan-calc` | 1.00 | -| `court-form-filling` | 1.00 | +| `court-form-filling` | TBD (re-baseline after #234) | | `dialogue-parser` | ~0.83 | -The court-form score above predates the verifier fix for issue #234 and the -pypdf update. Those historical scores are not comparable with scores from -the updated sample. - ## How it works (brief) - **`environment/Dockerfile`** rebases the upstream task onto `coder-eval-agent` and bakes the inputs at the paths the prompt expects. diff --git a/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml b/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml index 5a104ef56..0a09fd16e 100644 --- a/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml +++ b/tasks/samples/skillsbench/court-form-filling/court-form-filling.yaml @@ -46,9 +46,13 @@ run_limits: turn_timeout: 300 success_criteria: - # Run the skillsbench verifier verbatim: copy it from the task dir into a - # writable /verifier, run its test.sh (writes /logs/verifier/reward.txt), and - # emit that reward (0.0-1.0) as the criterion score. + # Run the skillsbench verifier (patched locally to fail on missing checkbox + # data or form fields, #234): copy it from the task dir into a writable + # /verifier, run its test.sh (writes /logs/verifier/reward.txt), and emit + # that reward (0.0-1.0) as the criterion score. + # Grading still runs in the agent's container, so the pytest on PATH, + # auto-loaded pytest11 plugins, site-packages and the $TASK_DIR verifier + # copy remain tamperable; see the follow-up issue #238. - type: run_command command: | set -e diff --git a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py index 9b226c893..2239e3e38 100644 --- a/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py +++ b/tasks/samples/skillsbench/court-form-filling/verifier/test_outputs.py @@ -1,4 +1,4 @@ -# Modified from benchflow-ai/skillsbench@95b3e60: require form fields, preserve AcroForm read and field dereference errors, and expose extract_field_values (issue #234). +# Modified from benchflow-ai/skillsbench@95b3e60: fail on missing form data (issue #234). """ Tests for court-form-filling task (SC-100 Small Claims Court form). @@ -12,6 +12,7 @@ import re import subprocess +import warnings import xml.etree.ElementTree as ET from pathlib import Path @@ -26,8 +27,10 @@ def extract_xfa_xml(pdf_path: Path) -> str: Extract XFA XML datasets from a PDF file. Returns the raw XML string or empty string if not found. """ + from pypdf import PdfReader + from pypdf.errors import PdfReadError + try: - from pypdf import PdfReader reader = PdfReader(str(pdf_path)) # Try to access XFA data @@ -43,8 +46,8 @@ def extract_xfa_xml(pdf_path: Path) -> str: if stream_name == "datasets": stream_obj = xfa[i + 1].get_object() return stream_obj.get_data().decode("utf-8", errors="ignore") - except Exception: - pass + except (KeyError, PdfReadError) as exc: + warnings.warn(f"XFA datasets could not be read; using AcroForm checkboxes only: {exc!r}", stacklevel=2) return "" @@ -118,8 +121,8 @@ def find_checkboxes(element, path=""): find_checkboxes(child, current_path) find_checkboxes(root) - except Exception: - pass + except ET.ParseError as exc: + warnings.warn(f"XFA datasets did not parse; using AcroForm checkboxes only: {exc}", stacklevel=2) return checkbox_values @@ -338,14 +341,14 @@ def pdf_field_values(): return extract_field_values(OUTPUT_FILE) -def extract_field_values(pdf_path: Path) -> dict: +def extract_field_values(pdf_path: Path) -> dict[str, str]: """Return every AcroForm field value as a string ("" when unset).""" from pypdf import PdfReader reader = PdfReader(str(pdf_path)) fields = reader.get_fields() or {} - field_values = {} + field_values: dict[str, str] = {} for name, field in fields.items(): value = field.get("/V", "") if value: diff --git a/tests/lint/rules/ce071_patched_verifier_no_skip.py b/tests/lint/rules/ce071_patched_verifier_no_skip.py new file mode 100644 index 000000000..ad6eb51a3 --- /dev/null +++ b/tests/lint/rules/ce071_patched_verifier_no_skip.py @@ -0,0 +1,61 @@ +"""CE071: A locally patched binary-reward verifier must not skip or xfail. + +Flags ``pytest.skip`` / ``pytest.importorskip`` / ``pytest.xfail`` calls and +``pytest.mark.skip`` / ``skipif`` / ``xfail`` marks. A binary verifier's runner +writes reward 1 when pytest exits 0, so a skipped check counts as a pass. + +Scope: every ``tasks/samples/skillsbench/*/verifier/test_outputs.py`` whose first +line starts with ``PATCHED_MARKER``. The test runs this rule together with CE005, +which catches the broad ``except Exception: pass`` swallow. ``dialogue-parser`` is +out of scope because it is unpatched and its runner scores passed / total, so a +skip lowers its reward instead of passing it. + +BLIND SPOT: aliased imports (``from pytest import skip``) and dynamically built +calls are not resolved. + +Add ``# noqa: CE071`` on the offending line for a deliberate exception. + +Rationale: .claude/notes/lint-rules.md § CE071 +""" + +import ast + +from tests.lint.rules.base import BaseRule + + +PATCHED_MARKER = "# Modified from benchflow-ai/skillsbench@" +_OUTCOME_CALLS = frozenset({"skip", "importorskip", "xfail"}) +_SKIP_MARKS = frozenset({"skip", "skipif", "xfail"}) + + +def _is_pytest_name(node: ast.expr) -> bool: + return isinstance(node, ast.Name) and node.id == "pytest" + + +def _is_pytest_outcome_call(node: ast.Call) -> bool: + func = node.func + return isinstance(func, ast.Attribute) and func.attr in _OUTCOME_CALLS and _is_pytest_name(func.value) + + +def _is_pytest_skip_mark(node: ast.Attribute) -> bool: + owner = node.value + return ( + node.attr in _SKIP_MARKS + and isinstance(owner, ast.Attribute) + and owner.attr == "mark" + and _is_pytest_name(owner.value) + ) + + +class PatchedVerifierNoSkip(BaseRule): + id = "CE071" + + def visit_Call(self, node: ast.Call) -> None: + if _is_pytest_outcome_call(node): + self.violation(node, "patched binary verifier must fail, not skip or xfail") + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> None: + if _is_pytest_skip_mark(node): + self.violation(node, "patched binary verifier must not carry a skip or xfail mark") + self.generic_visit(node) diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 7eb1e52df..bb1a7289e 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -5403,68 +5403,60 @@ def test_shares_manifest_skill_dirs_with_the_runtime_allowlist(self): @pytest.mark.lint class TestCE071PatchedBinaryVerifiersDoNotSkip: - """CE071: patched binary verifiers must not skip, xfail, or catch BaseException. + """CE071 + CE005 over every SkillsBench verifier that carries the patch marker. - Scope: the locally patched 3d-scan-calc and court-form-filling verifiers. - Blind spots: imported aliases and dynamically constructed calls are not resolved. + The rule and its scope are documented in ``tests/lint/rules/ce071_patched_verifier_no_skip.py``. """ @staticmethod - def _offenders(source: str) -> list[int]: - import ast - - return [ - node.lineno - for node in ast.walk(ast.parse(source)) - if ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and isinstance(node.func.value, ast.Name) - and node.func.value.id == "pytest" - and node.func.attr in {"skip", "importorskip", "xfail"} - ) - or ( - isinstance(node, ast.Attribute) - and node.attr in {"skip", "skipif", "xfail"} - and isinstance(node.value, ast.Attribute) - and node.value.attr == "mark" - and isinstance(node.value.value, ast.Name) - and node.value.value.id == "pytest" - ) - or ( - isinstance(node, ast.ExceptHandler) - and (node.type is None or (isinstance(node.type, ast.Name) and node.type.id == "BaseException")) - ) - ] + def _patched_verifiers() -> list[Path]: + from tests.lint.rules.ce071_patched_verifier_no_skip import PATCHED_MARKER - @pytest.mark.parametrize("slug", ["3d-scan-calc", "court-form-filling"]) - def test_patched_verifier(self, slug: str) -> None: - path = ( - Path(__file__).resolve().parents[1] - / "tasks" - / "samples" - / "skillsbench" - / slug - / "verifier/test_outputs.py" + root = SRC.parent / "tasks" / "samples" / "skillsbench" + return sorted( + path + for path in root.glob("*/verifier/test_outputs.py") + if path.read_text(encoding="utf-8").startswith(PATCHED_MARKER) ) - offenders = self._offenders(path.read_text(encoding="utf-8")) - assert not offenders, f"CE071: silent outcome or broad catch in {path} at lines {offenders}" + + @staticmethod + def _run(tmp_path: Path, source: str) -> list[str]: + from tests.lint.rules.ce071_patched_verifier_no_skip import PatchedVerifierNoSkip + from tests.lint.rules.no_silent_except import NoSilentExcept + + path = tmp_path / "test_outputs.py" + path.write_text(source, encoding="utf-8") + return [v.rule_id for v in check_paths([path], rules=[PatchedVerifierNoSkip, NoSilentExcept])] + + def test_patched_verifiers_are_found(self) -> None: + names = {path.parts[-3] for path in self._patched_verifiers()} + assert {"3d-scan-calc", "court-form-filling"} <= names + + def test_patched_verifiers_clean(self) -> None: + from tests.lint.rules.ce071_patched_verifier_no_skip import PatchedVerifierNoSkip + from tests.lint.rules.no_silent_except import NoSilentExcept + + violations = check_paths(self._patched_verifiers(), rules=[PatchedVerifierNoSkip, NoSilentExcept]) + assert not violations, "\n".join(str(v) for v in violations) @pytest.mark.parametrize( - "source", + ("source", "rule_id"), [ - "pytest.skip('missing')", - "pytest.importorskip('pypdf')", - "pytest.xfail('missing')", - "@pytest.mark.skip\ndef test(): pass", - "@pytest.mark.skipif(True, reason='missing')\ndef test(): pass", - "@pytest.mark.xfail\ndef test(): pass", - "try: read()\nexcept: pass", - "try: read()\nexcept BaseException: pass", + ("pytest.skip('missing')", "CE071"), + ("pytest.importorskip('pypdf')", "CE071"), + ("pytest.xfail('missing')", "CE071"), + ("@pytest.mark.skip\ndef test(): pass", "CE071"), + ("@pytest.mark.skipif(True, reason='missing')\ndef test(): pass", "CE071"), + ("@pytest.mark.xfail\ndef test(): pass", "CE071"), + ("try: read()\nexcept: pass", "CE005"), + ("try: read()\nexcept Exception: pass", "CE005"), ], ) - def test_detects_silent_outcomes(self, source: str) -> None: - assert self._offenders(source) + def test_detects_silent_outcomes(self, tmp_path: Path, source: str, rule_id: str) -> None: + assert self._run(tmp_path, source) == [rule_id] + + def test_allows_failure_and_narrow_exception(self, tmp_path: Path) -> None: + assert not self._run(tmp_path, "try: read()\nexcept ValueError: pytest.fail('invalid')") - def test_allows_failure_and_narrow_exception(self) -> None: - assert not self._offenders("try: read()\nexcept ValueError: pytest.fail('invalid')") + def test_honours_noqa(self, tmp_path: Path) -> None: + assert not self._run(tmp_path, "pytest.skip('optional') # noqa: CE071") diff --git a/tests/test_skillsbench_court_form_verifier.py b/tests/test_skillsbench_court_form_verifier.py index ab884963c..0205df99e 100644 --- a/tests/test_skillsbench_court_form_verifier.py +++ b/tests/test_skillsbench_court_form_verifier.py @@ -4,7 +4,10 @@ import importlib.util import re +import subprocess +import sys import tomllib +import xml.etree.ElementTree as ET from pathlib import Path from unittest.mock import Mock @@ -21,61 +24,71 @@ _VERIFIER = importlib.util.module_from_spec(_SPEC) _SPEC.loader.exec_module(_VERIFIER) +pytestmark = pytest.mark.filterwarnings("ignore:XFA datasets:UserWarning") -@pytest.fixture(autouse=True) -def reject_verifier_skips(monkeypatch: pytest.MonkeyPatch) -> None: - """A direct verifier call must fail if it tries to skip a check.""" - monkeypatch.setattr(pytest, "skip", pytest.fail) +type RequiredCase = tuple[str, str, str, bool] +type NamedCase = tuple[str, str] -def test_verifier_skip_fails() -> None: - with pytest.raises(pytest.fail.Exception, match="missing"): - pytest.skip("missing") +def _write_correct_fill(output: Path) -> Path: + reader = PdfReader(_BLANK_PDF) + fields = reader.get_fields() + assert fields is not None + values = { + name: expected + for pattern, _, expected, _ in _VERIFIER.REQUIRED_CHECKBOXES + for name, field in fields.items() + if pattern in name and expected in field.get("/_States_", []) + } + writer = PdfWriter(clone_from=reader) + writer.update_page_form_field_values(list(writer.pages), values, auto_regenerate=False) + writer.write(output) + return output @pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) -def test_required_checkbox_missing_fails(case: tuple) -> None: +def test_required_checkbox_missing_fails(case: RequiredCase) -> None: with pytest.raises(pytest.fail.Exception, match="not found"): _VERIFIER.TestCheckboxes().test_checkbox_value({}, *case) @pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) -def test_required_checkbox_wrong_value_fails(case: tuple) -> None: +def test_required_checkbox_wrong_value_fails(case: RequiredCase) -> None: with pytest.raises(AssertionError, match="expected value"): _VERIFIER.TestCheckboxes().test_checkbox_value({case[0]: "wrong"}, *case) @pytest.mark.parametrize("case", _VERIFIER.REQUIRED_CHECKBOXES, ids=lambda case: case[1]) -def test_required_checkbox_correct_value_passes(case: tuple) -> None: +def test_required_checkbox_correct_value_passes(case: RequiredCase) -> None: _VERIFIER.TestCheckboxes().test_checkbox_value({case[0]: case[2]}, *case) @pytest.mark.parametrize("case", _VERIFIER.UNCHECKED_CHECKBOXES, ids=lambda case: case[1]) -def test_unchecked_checkbox_absent_passes(case: tuple) -> None: +def test_unchecked_checkbox_absent_passes(case: NamedCase) -> None: _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked({}, *case) @pytest.mark.parametrize("case", _VERIFIER.UNCHECKED_CHECKBOXES, ids=lambda case: case[1]) -def test_unchecked_checkbox_checked_fails(case: tuple) -> None: +def test_unchecked_checkbox_checked_fails(case: NamedCase) -> None: with pytest.raises(AssertionError, match="should be unchecked"): _VERIFIER.TestUncheckedCheckboxes().test_checkbox_unchecked({case[0]: "/1"}, *case) @pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) -def test_empty_field_missing_fails(case: tuple) -> None: +def test_empty_field_missing_fails(case: NamedCase) -> None: values = {name: "" for name, _ in _VERIFIER.EMPTY_FIELDS if name != case[0]} with pytest.raises(AssertionError, match="not found"): _VERIFIER.TestEmptyFields().test_field_empty(values, *case) @pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) -def test_empty_field_filled_fails(case: tuple) -> None: +def test_empty_field_filled_fails(case: NamedCase) -> None: with pytest.raises(AssertionError, match="should be empty"): _VERIFIER.TestEmptyFields().test_field_empty({case[0]: "filled"}, *case) @pytest.mark.parametrize("case", _VERIFIER.EMPTY_FIELDS, ids=lambda case: case[1]) -def test_empty_field_blank_passes(case: tuple) -> None: +def test_empty_field_blank_passes(case: NamedCase) -> None: _VERIFIER.TestEmptyFields().test_field_empty({case[0]: ""}, *case) @@ -92,19 +105,7 @@ def test_expected_fields_exist_in_shipped_pdf() -> None: def test_correct_pdf_field_roundtrip_passes(tmp_path: Path) -> None: - reader = PdfReader(_BLANK_PDF) - fields = reader.get_fields() - assert fields is not None - values = { - name: expected - for pattern, _, expected, _ in _VERIFIER.REQUIRED_CHECKBOXES - for name, field in fields.items() - if pattern in name and expected in field.get("/_States_", []) - } - output = tmp_path / "filled.pdf" - writer = PdfWriter(clone_from=reader) - writer.update_page_form_field_values(list(writer.pages), values, auto_regenerate=False) - writer.write(output) + output = _write_correct_fill(tmp_path / "filled.pdf") checkboxes = _VERIFIER.extract_checkbox_values(output) field_values = _VERIFIER.extract_field_values(output) for case in _VERIFIER.REQUIRED_CHECKBOXES: @@ -149,13 +150,63 @@ def test_field_extraction_preserves_dereference_error(monkeypatch: pytest.Monkey _VERIFIER.extract_field_values(_BLANK_PDF) -def test_dockerfile_pypdf_pin_matches_dev_pin() -> None: - """The regression tests above only vouch for the pypdf the verifier image installs.""" - dev = tomllib.loads((_REPO / "pyproject.toml").read_text(encoding="utf-8"))["project"]["optional-dependencies"][ - "dev" - ] - dev_pins = [re.fullmatch(r"pypdf(?:\[[^]]*\])?==(\S+)", dep) for dep in dev] - dockerfile_pins = re.findall( - r"\bpypdf==(\S+)", (_TASK_DIR / "environment" / "Dockerfile").read_text(encoding="utf-8") +def test_shipped_xfa_datasets_do_not_parse_and_warn() -> None: + """Pins the upstream contract: checkbox grading uses AcroForm values only. + + The namespace cleanup leaves ``xfa:`` attribute prefixes, so the shipped datasets never parse. Making them + parse would grade XFA values too (a blank ``"0"`` matches unchecked patterns such as ``Checkbox11``), which + is a separate grading change. + """ + xfa_xml = _VERIFIER.extract_xfa_xml(_BLANK_PDF) + assert xfa_xml + with pytest.warns(UserWarning, match="XFA datasets did not parse.*unbound prefix"): + assert _VERIFIER.parse_xfa_checkboxes(xfa_xml) == {} + + +def _run_verifier(tmp_path: Path, output_pdf: Path) -> tuple[int, ET.Element]: + """Run the verifier as test.sh does, with OUTPUT_FILE pointed at ``output_pdf``.""" + source = (_TASK_DIR / "verifier" / "test_outputs.py").read_text(encoding="utf-8") + original = 'OUTPUT_FILE = Path("/root/sc100-filled.pdf")' + assert source.count(original) == 1 + verifier = tmp_path / "test_outputs.py" + verifier.write_text(source.replace(original, f"OUTPUT_FILE = Path({str(output_pdf)!r})"), encoding="utf-8") + (tmp_path / "pytest.ini").write_text("[pytest]\n", encoding="utf-8") + junit = tmp_path / "junit.xml" + command = [sys.executable, "-m", "pytest", str(verifier), "-k", "Checkboxes or EmptyFields"] + proc = subprocess.run( + [*command, "-p", "no:cacheprovider", "-c", str(tmp_path / "pytest.ini"), f"--junitxml={junit}"], + cwd=tmp_path, + capture_output=True, + text=True, + encoding="utf-8", + check=False, ) - assert [m.group(1) for m in dev_pins if m] == dockerfile_pins + suite = ET.parse(junit).getroot().find("testsuite") + assert suite is not None, proc.stdout + return proc.returncode, suite + + +def test_verifier_run_fails_blank_form_without_skips(tmp_path: Path) -> None: + returncode, suite = _run_verifier(tmp_path, _BLANK_PDF) + assert returncode != 0 + assert int(suite.get("failures", "0")) > 0 + assert suite.get("skipped") == "0" + + +def test_verifier_run_passes_correct_fill(tmp_path: Path) -> None: + output = _write_correct_fill(tmp_path / "filled.pdf") + returncode, suite = _run_verifier(tmp_path, output) + assert returncode == 0 + assert int(suite.get("tests", "0")) > 0 + assert suite.get("skipped") == "0" + + +def test_dockerfile_pins_match_lock() -> None: + """The regression tests above only vouch for the PDF stack the verifier image installs.""" + dockerfile = (_TASK_DIR / "environment" / "Dockerfile").read_text(encoding="utf-8") + locked = { + package["name"]: package["version"] + for package in tomllib.loads((_REPO / "uv.lock").read_text(encoding="utf-8"))["package"] + } + for name in ("pypdf", "cryptography"): + assert re.findall(rf"\b{name}==(\S+)", dockerfile) == [locked[name]], name diff --git a/uv.lock b/uv.lock index d93d843d9..34898ad2e 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.13" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -637,7 +637,7 @@ requires-dist = [ { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.5.1" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, - { name = "pypdf", extras = ["crypto"], marker = "extra == 'dev'", specifier = "==7.0.1" }, + { name = "pypdf", extras = ["crypto"], marker = "extra == 'dev'", specifier = ">=7.0.1" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.408" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" },