diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71cb3cbf..b7b7e270 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,10 @@ on: jobs: ci: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.13", "3.14"] services: postgres: image: postgres:16-alpine @@ -24,6 +28,8 @@ jobs: --health-retries=5 env: + # Override .python-version so uv uses the interpreter selected by this job. + UV_PYTHON: ${{ matrix.python-version }} DB_HOST: 127.0.0.1 DB_PORT: "5432" DB_USER: agent_control @@ -37,7 +43,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} - name: Setup uv uses: astral-sh/setup-uv@v4 @@ -59,8 +65,25 @@ jobs: run: make sync - name: Run Python checks + if: matrix.python-version != '3.14' run: make check + - name: Build and smoke test Quarry on Python 3.13 + if: matrix.python-version == '3.13' + run: make -C evaluators/contrib/quarry build example + + - name: Run Python 3.14 checks (Quarry blocked by upstream metadata) + if: matrix.python-version == '3.14' + run: | + make check -o contrib-test -o contrib-lint -o contrib-typecheck + for package in $(uv run python scripts/contrib_packages.py names); do + if [ "$package" = "quarry" ]; then + echo "Quarry excluded: cisco-project-quarry==0.1.0 requires Python <3.14." + continue + fi + make -C "evaluators/contrib/$package" check + done + - name: Collect coverage files id: coverage_files run: | diff --git a/evaluators/builtin/pyproject.toml b/evaluators/builtin/pyproject.toml index 47dcf697..c66301fc 100644 --- a/evaluators/builtin/pyproject.toml +++ b/evaluators/builtin/pyproject.toml @@ -19,6 +19,7 @@ galileo = ["agent-control-evaluator-galileo>=8.6.0"] budget = ["agent-control-evaluator-budget>=7.5.0"] cisco = ["agent-control-evaluator-cisco>=7.5.0"] defenseclaw = ["agent-control-evaluator-defenseclaw>=8.2.0"] +quarry = ["agent-control-evaluator-quarry>=8.15.0; python_version < '3.14'"] dev = ["pytest>=8.0.0", "pytest-asyncio>=0.23.0"] [project.entry-points."agent_control.evaluators"] @@ -41,3 +42,4 @@ agent-control-evaluator-galileo = { path = "../contrib/galileo", editable = true agent-control-evaluator-budget = { path = "../contrib/budget", editable = true } agent-control-evaluator-cisco = { path = "../contrib/cisco", editable = true } agent-control-evaluator-defenseclaw = { path = "../contrib/defenseclaw", editable = true } +agent-control-evaluator-quarry = { path = "../contrib/quarry", editable = true } diff --git a/evaluators/builtin/tests/test_contrib_packages.py b/evaluators/builtin/tests/test_contrib_packages.py index 3ae1670f..6114a0de 100644 --- a/evaluators/builtin/tests/test_contrib_packages.py +++ b/evaluators/builtin/tests/test_contrib_packages.py @@ -41,6 +41,7 @@ def test_discover_contrib_packages_returns_expected_metadata() -> None: ("cisco", "agent-control-evaluator-cisco", "cisco"), ("defenseclaw", "agent-control-evaluator-defenseclaw", "defenseclaw"), ("galileo", "agent-control-evaluator-galileo", "galileo"), + ("quarry", "agent-control-evaluator-quarry", "quarry"), ] diff --git a/evaluators/contrib/README.md b/evaluators/contrib/README.md index 5ae14c6a..3c6f490e 100644 --- a/evaluators/contrib/README.md +++ b/evaluators/contrib/README.md @@ -3,6 +3,7 @@ Contributed evaluators and templates for extending Agent Control. - `galileo/` — Galileo evaluator integrations (Luna, Decision Models, LLM-as-judge) +- `quarry/` — Local Project Quarry policies over trace/session steps (Python 3.12–3.13) - `template/` — Starter template for adding new evaluators Full guide: https://docs.agentcontrol.dev/concepts/evaluators/custom-evaluators diff --git a/evaluators/contrib/quarry/Makefile b/evaluators/contrib/quarry/Makefile new file mode 100644 index 00000000..b15080b4 --- /dev/null +++ b/evaluators/contrib/quarry/Makefile @@ -0,0 +1,24 @@ +.PHONY: sync test lint lint-fix typecheck check build example + +sync: + uv sync + +test: + uv run pytest tests --cov=src --cov-report=xml:../../../coverage-evaluators-quarry.xml -q + +lint: + uv run ruff check --config ../../../pyproject.toml src/ tests/ examples/ + +lint-fix: + uv run ruff check --config ../../../pyproject.toml --fix src/ tests/ examples/ + +typecheck: + uv run mypy --config-file ../../../pyproject.toml src/ + +check: test lint typecheck + +build: + uv build + +example: + uv run python examples/local_policy.py diff --git a/evaluators/contrib/quarry/PYTHON314.md b/evaluators/contrib/quarry/PYTHON314.md new file mode 100644 index 00000000..f5ada7f2 --- /dev/null +++ b/evaluators/contrib/quarry/PYTHON314.md @@ -0,0 +1,201 @@ +# Python 3.14 compatibility audit + +Audit date: November 9, 2026. Branch: `feature/quarry-for-controls`. +Implementation baseline: `aff76246c9b003ab3553ea6fa32b5bbc880d0367`. + +## Supported deployment boundary + +Python **3.13** is the supported integration target for Orbit deployments that +load the Quarry evaluator. Both `agent-control-evaluator-quarry==8.15.0` and its +pinned upstream dependency `cisco-project-quarry==0.1.0` admit that interpreter. +They can be installed with normal metadata resolution; no fork, metadata bypass, +or upstream release is needed for this target. Python 3.12 is also admitted. + +Agent Control core continues supporting Python 3.14 **without Quarry**. A process +loading Quarry must use a supported interpreter throughout its image, toolchain, +and dependency environment. A source pin alone does not make an in-process +Python 3.14 Quarry installation supported. Orbit's image/toolchain update belongs in +Orbit; this repository supplies the compatible evaluator and CI coverage. + +CI now exercises 3.12, 3.13, and 3.14. The 3.13 job runs the complete `make check` +including Quarry, engine, and builtin evaluator tests, then builds Quarry's +wheel/sdist and runs the offline example. The 3.14 job continues checking core +and other contrib packages with an explicit Quarry exclusion. + +## Python 3.13 normal installation validation + +On CPython 3.13.13, the following completed with normal metadata resolution and +without `UV_NO_SYNC`, `--ignore-requires-python`, or dependency overrides: + +```sh +UV_PYTHON=3.13 make -C evaluators/contrib/quarry sync check example build +``` + +All 54 Quarry tests passed, along with lint, typecheck, the real offline scan +example, and wheel/sdist builds. The resulting 40-package development environment +passed `uv pip check`; all installed distributions admit the interpreter. + +A clean source snapshot also passed the complete root `make sync check` under +Python 3.13.13, without excluding any target: 2,928 tests passed and two SDK +tests skipped because their optional Galileo extras were absent. This includes +138 engine, 311 builtin evaluator, and 54 Quarry tests, plus all repository lint +and typechecks. Each contrib package used its own normally resolved environment. + +A separate clean environment installed the built builtin evaluator and models +wheels with `agent-control-evaluators[quarry]`, resolving Quarry from its built +wheel. All 27 runtime distributions passed dependency and interpreter metadata +checks. Entry-point discovery and an actual matching policy scan succeeded from +installed `site-packages`, with no editable source imports. + +Built metadata matches the manifests: our Quarry wheel retains +`Requires-Python: >=3.12,<3.14` and `cisco-project-quarry==0.1.0`. The builtin extra +selects Quarry for Python 3.13 and excludes it for Python 3.14. This validates the +supported Python 3.13 install path without changing the per-package boundary. + +The same normal `sync check example build` targets also passed in a native Linux +ARM64 container on CPython 3.13.18: 54 tests, lint, typecheck, a matching scan, and +wheel/sdist builds. All 40 installed development distributions passed dependency +checks and an explicit check of their `Requires-Python` bounds. No metadata was +bypassed. Orbit's actual image/Bazel integration remains a downstream check. + +## Python 3.14 Quarry integration remains blocked + +There is no supported Python 3.14 Quarry dependency to pin yet. The only published +release, `cisco-project-quarry==0.1.0`, declares `Requires-Python: >=3.11,<3.14`. +Upstream `main` also excludes 3.14. Sources: +[release metadata](https://pypi.org/pypi/cisco-project-quarry/0.1.0/json), +[upstream manifest](https://github.com/cisco-ai-defense/project-quarry/blob/main/pyproject.toml). + +Agent Control's Quarry package correctly retains `>=3.12,<3.14`, and the builtin +`quarry` extra still excludes Python 3.14. Removing only those two restrictions +would leave an incompatible upstream distribution. An upstream release or an +approved immutable compatibility fork is needed before widening our metadata. +Do not treat the baseline SHA as an Orbit-compatible Python 3.14 source pin. + +## Workspace metadata, builds, and automation + +| Area | Finding | +| --- | --- | +| Workspace root and server | `>=3.12,<3.15`; already admit 3.14. | +| Models, engine, telemetry, builtin evaluators, Python SDK | `>=3.12`; already admit 3.14. | +| Budget, Cisco, DefenseClaw, Galileo contrib packages | `>=3.12`; install and pass checks on 3.14. | +| Quarry contrib | `>=3.12,<3.14`; normal `make sync` rejects a 3.14 interpreter. | +| Standalone example manifests | All declare `>=3.12`; no additional metadata exclusion. Their separate framework/application environments were not all executed. | +| Classifiers | The SDK was the only package listing individual Python versions; 3.14 was added after its tests passed. | +| Interpreter selection | `.python-version` remains `3.12.13`. Checks explicitly set `UV_PYTHON=3.14`; changing `setup-python` alone would not override this file. | +| CI | Matrix covers 3.12, 3.13, and 3.14. The 3.13 job checks and builds Quarry normally; 3.14 explicitly excludes Quarry. | +| Release builds | Python publishing and the TypeScript release helper use 3.12. Their packaging does not itself forbid consumers on 3.14. The existing release defaults remain unchanged. | +| Server Docker image | Uses `ghcr.io/astral-sh/uv:python3.12-bookworm-slim`; its default was not changed. Orbit's separate image and Bazel toolchain were not modified or validated here. | +| Bazel | This repository has no Bazel configuration to update. | +| Wheels/sdists | Root `make build` successfully built core and all contrib packages under 3.14. Building a universal wheel does not establish compatible `Requires-Python` metadata. | + +## Dependency closure + +The evaluator's runtime closure was traversed with active extras, including +`sqlglot[c]`. All 23 third-party dependencies below install under CPython 3.14.7 +on macOS ARM64. Only our Quarry package and upstream Quarry fail the interpreter +metadata check. These were scratch environments; no production requirement or +upstream wheel metadata was rewritten. + +```text +annotated-types==0.8.0 attrs==26.1.0 +cachetools==7.2.0 google-re2==1.1.20251105 +jsonschema==4.26.0 jsonschema-specifications==2025.9.1 +lark==1.3.1 loguru==0.7.3 +markdown-it-py==4.2.0 mdurl==0.1.2 +orjson==3.13.0 packaging==26.3 +pydantic==2.14.1 pydantic-core==2.50.1 +pygments==2.21.0 pyyaml==6.0.3 +referencing==0.37.0 rich==15.0.0 +rpds-py==2026.9.1 sqlglot==30.11.0 +sqlglotc==30.11.0 typing-extensions==4.16.0 +typing-inspection==0.4.5 +``` + +Binary-only resolution of this complete closure also succeeded for Python 3.14 +on Linux x86-64 (`x86_64-manylinux_2_28`). This checks wheel availability, not +execution on that architecture. An attempted Linux x86-64 container run failed +in QEMU while starting the tooling, before Quarry checks could execute. + +Native Linux ARM64 diagnostics also passed on CPython 3.14.10: all 54 Quarry +tests, lint, typecheck, offline example, and wheel/sdist build. That disposable +container explicitly used pip's `--ignore-requires-python` and uv's +`UV_NO_SYNC=1`. Its installed upstream metadata still excluded 3.14. The +container's uv dependency checker did not flag that Python restriction, so the +audit also checked it explicitly with `packaging.specifiers.SpecifierSet`; +the result was incompatible. Installer success is not support certification. + +## Checks performed + +Existing Makefile targets were used with isolated `/tmp` environments and public +PyPI. The normal workspace installation contains no Quarry on 3.14. PostgreSQL +16 was available for the server and SDK integration suites. + +| CPython 3.14.7 on macOS ARM64 | Result | +| --- | --- | +| Workspace `make sync` and `uv pip check` | Passed; all installed packages compatible. | +| Models | 76 passed. | +| Telemetry | 20 passed. | +| Engine | 138 passed. | +| Builtin evaluators | 311 passed. | +| Server | 1,040 passed. | +| Python SDK | 754 passed, 2 skipped because optional Galileo extras were absent. | +| Root packaging/build scripts | 6 passed. | +| Budget / Cisco / DefenseClaw / Galileo | 122 / 23 / 26 / 358 passed; package lint and typecheck passed. | +| Core lint | Passed. | +| Core typecheck with mypy 2.4.1 | All 116 source files passed. | +| Quarry, diagnostic installation only | 54 passed; lint, typecheck, offline example, and wheel/sdist build passed. | +| Normal Quarry installation | Blocked by `>=3.12,<3.14`. | +| macOS diagnostic environment dependency check | Fails as expected for local Quarry and `cisco-project-quarry`; runtime success does not remove this incompatibility. | + +One server test had assumed that valid JSON nested 10,000 levels always raises +a decoder recursion error. Python 3.14 can decode it. The test now accepts either +an accurate parsed count or an undecodable summary, while requiring the same +502 rejection and empty sanitized error details. All 1,040 server tests also +passed on Python 3.12.13 after this test-only adjustment. Server runtime behavior +was not changed. + +## Lockfiles and reproducibility + +Python `uv.lock` files are ignored by this repository, including the root and +contrib lockfiles. No Python lock is published by this change. The local root +lock admits `>=3.12,<3.15`; Quarry's local lock retains `>=3.12,<3.14`. + +The existing local root lock selected mypy 2.1.0, which reports two `TypeAdapter` +alias errors with Pydantic 2.14.1. The same errors reproduce on 3.12, so they are +not a Python 3.14 runtime regression. Mypy 2.4.1 passes the entire core tree. +A clean dependency resolution was tested separately from the existing local +lock; this distinction matters when reproducing CI from an ignored lockfile. +That clean resolution selected 89 installed packages, including mypy 2.4.1 and +Pydantic 2.14.1, and passed the complete core check command: + +```sh +UV_INDEX=https://pypi.org/simple UV_PYTHON=3.14 \ + UV_PROJECT_ENVIRONMENT=/tmp/agent-control-clean314-env \ + make check -o contrib-test -o contrib-lint -o contrib-typecheck +``` + +The non-Quarry contrib packages were checked independently in separate 3.14 +environments. The exclusions above isolate their package environments and the +known Quarry blocker; they do not replace those independent contrib checks. +The CI YAML was parsed and its interpreter selection verified locally; the +hosted GitHub workflow itself has not been run as part of this audit. + +The documented diagnostic Quarry checks used `UV_NO_SYNC=1` after explicitly +installing the incompatible distributions into a disposable environment. This +is a compatibility experiment, not an installation recommendation. + +## Requirements for future Python 3.14 Quarry support + +These requirements concern a future 3.14 Quarry deployment. They do not block +the supported Python 3.13 deployment described above. + +1. Obtain a reviewed upstream release or immutable fork that declares Python + 3.14 support and passes the compiler/scanner tests and dependency checks. +2. Update our upstream dependency pin, widen the evaluator's Python range, and + update the builtin extra's interpreter marker together. +3. Re-run normal installs, all Quarry checks, and the complete workspace checks + without metadata bypasses or the explicit CI Quarry exclusion. Validate the + actual Orbit Linux/Bazel environment; x86-64 wheel resolution is insufficient. +4. Commit and normally push that validated change, then provide its full SHA. + The current implementation baseline does not satisfy these 3.14 conditions. diff --git a/evaluators/contrib/quarry/README.md b/evaluators/contrib/quarry/README.md new file mode 100644 index 00000000..47e108dd --- /dev/null +++ b/evaluators/contrib/quarry/README.md @@ -0,0 +1,131 @@ +# Project Quarry evaluator + +Optional `quarry.policy` evaluates one reviewed [Project Quarry](https://github.com/cisco-ai-defense/project-quarry) +DSL rule over Agent Control's recursive `Step`. A rule match triggers the control's +configured action. Local symbolic rules run without credentials, network calls, or +Galileo records. Model adapters, semantic probers, external text resources, compiled +byte uploads, rule parameters, and policy authoring are outside this integration. + +## Install + +Use Python **3.12 or 3.13**, matching Quarry's supported Python range. Python +**3.13** is the validated integration target for Orbit deployments that bundle +Quarry; Agent Control core can continue running on Python 3.14 without Quarry. + +```sh +pip install agent-control-evaluator-quarry +# Or the convenience extra, on Python 3.12–3.13: +pip install 'agent-control-evaluators[quarry]' +``` + +Install in each SDK/server environment that executes the control. The package is +discovered through the `agent_control.evaluators` entry point; core installations +do not load Quarry. The convenience extra excludes Quarry on Python 3.14 because +Quarry does not support it. Direct installation reports the Python incompatibility. +Quarry is alpha; this integration pins `cisco-project-quarry==0.1.0` for DSL stability. + +Python 3.14 remains blocked by that upstream release's `Requires-Python` metadata, +even though diagnostic runtime tests pass when metadata enforcement is bypassed. +See the [Python 3.14 compatibility audit](PYTHON314.md) for workspace checks, +dependency findings, and the requirements for an Orbit-compatible source pin. + +## Control configuration + +```json +{ + "description": "Detect reading a secret followed by an HTTP send", + "enabled": true, + "execution": "server", + "scope": {"step_types": ["trace", "session"], "stages": ["post"]}, + "condition": { + "selector": {"path": "*"}, + "evaluator": { + "name": "quarry.policy", + "config": { + "rule": "rule \"Read then send\"\ncondition:\n has_sequence(\"read_secret\", \"send_http\")\n", + "session_mode": "continuous", + "timeout_ms": 10000, + "max_messages": 10000 + } + } + }, + "action": {"decision": "deny"} +} +``` + +Submit a `trace` or `session` root `Step` through the existing evaluation API. +The evaluator uses the complete runtime step supplied by `evaluate_with_context`, +regardless of selector output. Direct `evaluate` accepts a `Step` or its serialized +object. No hierarchy is fetched from storage or accumulated between requests. +Choose one evaluation scope to avoid evaluating a trace twice, once alone and once +inside its session. + +## Step conversion and boundaries + +- Trace and session steps are containers. Put event content in `children`, in + chronological order; leave container `input`/`output` empty (`None`, `""`, `[]`, + or `{}`). Non-empty container summaries cause an error to avoid dropping or + double-counting events. Trace children may be `trace`, `llm`, or `tool` steps. +- LLM string input adds a user message; string output adds an assistant message. + Input may instead be an OpenAI message array, or `{"messages": [...]}`. Arrays + represent cumulative history and must extend the exact messages already emitted; + the shared prefix is included once. A structured output must be a complete + assistant message, including any `tool_calls`. Divergent/stale snapshots error; + use string input for an incremental user turn. LLM children are unsupported. +- A tool step becomes an assistant function call, followed by its nested children, + then a tool result if `output` is not `None`. Arguments stay structured through + JSON encoding; structured results become JSON text. For calls already emitted + by an LLM, set `context.tool_call_id` to the original ID. Name and arguments must + agree. Independent tool steps receive deterministic IDs; the converter never + guesses which pending assistant call a tool executes. +- `Step.tools` uses Quarry's OpenAI function-definition format. Definitions must + remain identical within each scan. Context fields other than `tool_call_id`, + names of containers/models, and `ground_truth` are not conversation messages. +- Unknown child types, nested sessions, ambiguous payloads, and hierarchies deeper + than 32 levels produce explicit errors. Quarry validates roles, tool links, + arguments, and its standard input/resource limits without truncation. + +Sessions require `session_mode`; there is no implicit default: + +| Mode | Meaning | +| --- | --- | +| `per_trace` | Scan each child trace independently; match if any trace matches, after every trace completes. Sequences cannot span traces. | +| `continuous` | Concatenate ordered child traces into one conversation. Sequences may span traces. Supply one session's events, with unique tool-call IDs and consistent tool definitions. | + +Sessions may only contain trace children. In continuous mode, later LLM histories +must include the earlier traces' conversation prefix. Use `per_trace` for independent +conversations or traces whose histories restart. Boundaries are not synthetic messages; +the policy sees no session/trace IDs. Coverage is limited to the events supplied. + +## Results, errors, and caching + +Successful results include `matched_rules`, `completion_state`, `scan_count`, the +policy digest, and the session mode when applicable. Empty input or a hierarchy +without messages returns a non-match without error. Invalid DSL or missing resources +reject evaluator construction. The existing evaluator factory caches each instance +by its full configuration; compilation occurs once per instance. Changing rule +source or limits creates another instance. Scans keep request state local. + +Invalid conversations, input-limit failures, timeouts, skipped rules, truncation, +and other incomplete scans return `matched=False`, confidence zero, and `error`. +Partial matches never become a decision, including when a later session trace fails. +The existing engine applies its configured error/action policy (deny controls fail +closed). Raw Quarry diagnostics and conversation content are omitted from errors. +The timeout covers the whole evaluation, including all independent session scans; +Quarry's input limits also apply. `max_messages` is a per-conversation limit. + +## Local development + +From the repository root: + +```sh +make -C evaluators/contrib/quarry sync +make -C evaluators/contrib/quarry check +make -C evaluators/contrib/quarry example +make -C evaluators/contrib/quarry build +make contrib-verify +``` + +The example runs fully offline after installation. Tests use the real pinned Quarry +compiler/scanner for matching, tool linkage, message histories, and session boundaries. +Upstream failure states are injected only for focused error-path coverage. diff --git a/evaluators/contrib/quarry/examples/local_policy.py b/evaluators/contrib/quarry/examples/local_policy.py new file mode 100644 index 00000000..08e86cd8 --- /dev/null +++ b/evaluators/contrib/quarry/examples/local_policy.py @@ -0,0 +1,57 @@ +"""Run an offline Quarry policy through Agent Control's cached evaluator factory.""" + +import asyncio + +from agent_control_evaluators import get_evaluator_instance +from agent_control_models import EvaluatorSpec, Step + +RULE = """rule "Read then send" +condition: + has_sequence("read_secret", "send_http") +""" + + +async def main() -> None: + guard = get_evaluator_instance( + EvaluatorSpec( + name="quarry.policy", + config={ + "rule": RULE, + # This session is an ordered conversation whose traces may be correlated. + "session_mode": "continuous", + }, + ) + ) + session = Step( + type="session", + name="demo", + input=None, + children=[ + Step( + type="trace", + name="read", + input=None, + children=[ + Step( + type="tool", name="read_secret", input={"path": "demo.txt"}, output="demo" + ), + ], + ), + Step( + type="trace", + name="send", + input=None, + children=[ + Step(type="tool", name="send_http", input={"body": "demo"}, output="sent"), + ], + ), + ], + ) + result = await guard.evaluate(session) + print(result.model_dump_json(indent=2)) + assert result.error is None + assert result.matched is True + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/evaluators/contrib/quarry/pyproject.toml b/evaluators/contrib/quarry/pyproject.toml new file mode 100644 index 00000000..6692d28b --- /dev/null +++ b/evaluators/contrib/quarry/pyproject.toml @@ -0,0 +1,41 @@ +[project] +name = "agent-control-evaluator-quarry" +version = "8.15.0" +description = "Local Project Quarry policy evaluator for Agent Control" +readme = "README.md" +requires-python = ">=3.12,<3.14" +license = { text = "Apache-2.0" } +authors = [{ name = "Agent Control Team" }] +dependencies = [ + "agent-control-evaluators>=8.15.0", + "agent-control-models>=8.15.0", + "cisco-project-quarry==0.1.0", +] + +[project.entry-points."agent_control.evaluators"] +"quarry.policy" = "agent_control_evaluator_quarry:QuarryEvaluator" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/agent_control_evaluator_quarry"] + +[tool.uv.sources] +agent-control-evaluators = { path = "../../builtin", editable = true } +agent-control-models = { path = "../../../models", editable = true } +agent-control-engine = { path = "../../../engine", editable = true } + +[dependency-groups] +dev = [ + "agent-control-engine>=8.15.0", + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "pytest-cov>=4.0.0", + "ruff>=0.1.0", + "mypy>=1.8.0", +] + +[tool.pytest.ini_options] +asyncio_default_fixture_loop_scope = "function" diff --git a/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/__init__.py b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/__init__.py new file mode 100644 index 00000000..3faa63f1 --- /dev/null +++ b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/__init__.py @@ -0,0 +1,6 @@ +"""Optional local Project Quarry evaluator.""" + +from .config import QuarryEvaluatorConfig +from .evaluator import QuarryEvaluator + +__all__ = ["QuarryEvaluator", "QuarryEvaluatorConfig"] diff --git a/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/config.py b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/config.py new file mode 100644 index 00000000..17d5a005 --- /dev/null +++ b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/config.py @@ -0,0 +1,30 @@ +"""Configuration for a single reviewed, local Quarry rule.""" + +from typing import Literal + +from agent_control_evaluators import EvaluatorConfig +from pydantic import ConfigDict, Field, field_validator + +type SessionMode = Literal["per_trace", "continuous"] + + +class QuarryEvaluatorConfig(EvaluatorConfig): + """Compile one symbolic rule; require an explicit boundary choice for sessions.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + rule: str = Field(min_length=1, description="Source of exactly one local Quarry DSL rule") + session_mode: SessionMode | None = Field( + default=None, + description="Required for sessions: independent traces, or one ordered conversation", + ) + timeout_ms: int = Field(default=10000, ge=1, le=300000, strict=True) + max_messages: int = Field(default=10000, ge=1, le=10000, strict=True) + + @field_validator("rule") + @classmethod + def nonblank_rule(cls, value: str) -> str: + """Reject whitespace without changing the reviewed rule source.""" + if not value.strip(): + raise ValueError("rule must contain Quarry DSL source") + return value diff --git a/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/conversion.py b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/conversion.py new file mode 100644 index 00000000..88f5ad71 --- /dev/null +++ b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/conversion.py @@ -0,0 +1,219 @@ +"""Translate recursive Agent Control steps into Quarry's OpenAI message wire format.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field + +from agent_control_models import JSONObject, JSONValue, Step + +from .config import SessionMode + + +class StepConversionError(ValueError): + """The supplied step cannot represent an unambiguous Quarry conversation.""" + + +def _text(value: JSONValue) -> str: + try: + return value if isinstance(value, str) else json.dumps(value, allow_nan=False) + except ValueError as exc: + raise StepConversionError("Tool data must be valid finite JSON.") from exc + + +def _empty(value: JSONValue) -> bool: + return value is None or value == "" or value == [] or value == {} + + +def _message_list(value: JSONValue) -> list[JSONObject]: + if isinstance(value, dict) and set(value) == {"messages"}: + value = value["messages"] + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise StepConversionError("LLM history must be a messages array of objects.") + return [item for item in value if isinstance(item, dict)] + + +@dataclass +class _ConversationBuilder: + """Request-local conversion state; never retained on the cached evaluator.""" + + max_messages: int + messages: list[JSONObject] = field(default_factory=list) + calls: dict[str, JSONObject] = field(default_factory=dict) + llm_calls: set[str] = field(default_factory=set) + tool_steps: set[str] = field(default_factory=set) + completed: set[str] = field(default_factory=set) + tools: list[JSONObject] | None = None + + def append(self, message: JSONObject, *, from_llm: bool = True) -> None: + if len(self.messages) >= self.max_messages: + raise StepConversionError("Conversation exceeds max_messages; no scan was run.") + self.messages.append(message) + calls = message.get("tool_calls") + if message.get("role") == "assistant" and isinstance(calls, list): + for call in calls: + if isinstance(call, dict) and isinstance(call.get("id"), str): + call_id = call["id"] + assert isinstance(call_id, str) + if call_id in self.calls: + raise StepConversionError("Duplicate tool-call ID in conversation.") + self.calls[call_id] = call + if from_llm: + self.llm_calls.add(call_id) + if message.get("role") == "tool": + call_id = message.get("tool_call_id") + if isinstance(call_id, str): + if call_id in self.completed: + raise StepConversionError("Duplicate tool result in conversation.") + self.completed.add(call_id) + + def add(self, step: Step, path: str, depth: int = 0) -> None: + if depth > 32: + raise StepConversionError("Step hierarchy exceeds 32 levels.") + if step.tools is not None: + if self.tools is not None and self.tools != step.tools: + raise StepConversionError("Tool definitions must remain identical within a scan.") + self.tools = step.tools + if step.type == "trace": + if not _empty(step.input) or not _empty(step.output): + raise StepConversionError( + "Trace content must be in Step.children, not input/output." + ) + for index, child in enumerate(step.children or []): + self.add(child, f"{path}_{index}", depth + 1) + elif step.type == "llm": + if step.children: + raise StepConversionError("LLM steps cannot have children; use a trace container.") + self.add_llm(step) + elif step.type == "tool": + self.add_tool(step, path, depth) + else: + raise StepConversionError("Supported child types are trace, llm, and tool.") + + def add_llm(self, step: Step) -> None: + if isinstance(step.input, str): + if step.input: + self.append({"role": "user", "content": step.input}) + elif step.input is not None: + history = _message_list(step.input) + # Message arrays represent cumulative history, while text represents a new turn. + # Require an exact prefix so repeated snapshots cannot invent duplicate events. + if history[: len(self.messages)] != self.messages: + raise StepConversionError( + "LLM history must extend the exact conversation prefix; " + "use text for a new turn." + ) + for message in history[len(self.messages) :]: + self.append(message) + if isinstance(step.output, str): + if step.output: + self.append({"role": "assistant", "content": step.output}) + elif step.output is not None: + if not isinstance(step.output, dict) or step.output.get("role") != "assistant": + raise StepConversionError("LLM output must be text or an assistant message object.") + self.append(step.output) + + def add_tool(self, step: Step, path: str, depth: int) -> None: + call_id = (step.context or {}).get("tool_call_id") + if call_id is not None and (not isinstance(call_id, str) or not call_id): + raise StepConversionError("context.tool_call_id must be a non-empty string.") + if call_id is None: + if self.llm_calls - self.completed - self.tool_steps: + raise StepConversionError( + "Tool steps linking an LLM call need context.tool_call_id." + ) + call_id = f"ac_quarry_{path}" + if call_id in self.calls: + raise StepConversionError("Generated tool-call ID conflicts with an existing call.") + assert isinstance(call_id, str) + if call_id in self.completed or call_id in self.tool_steps: + raise StepConversionError("Tool step repeats a tool-call ID.") + self.tool_steps.add(call_id) + if call_id in self.calls: + function = self.calls[call_id].get("function") + if not isinstance(function, dict) or function.get("name") != step.name: + raise StepConversionError("Tool step name does not match its assistant call.") + arguments = function.get("arguments") + try: + decoded: JSONValue = json.loads(arguments) if isinstance(arguments, str) else None + except ValueError as exc: + raise StepConversionError( + "Assistant tool-call arguments must be valid JSON." + ) from exc + if decoded != step.input: + raise StepConversionError("Tool step arguments do not match its assistant call.") + else: + self.append( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": {"name": step.name, "arguments": _text(step.input)}, + } + ], + }, + from_llm=False, + ) + for index, child in enumerate(step.children or []): + self.add(child, f"{path}_{index}", depth + 1) + if step.output is not None: + self.append({"role": "tool", "tool_call_id": call_id, "content": _text(step.output)}) + + def conversation(self) -> JSONObject: + result: JSONObject = {"messages": list(self.messages)} + if self.tools is not None: + result["tools"] = list(self.tools) + return result + + +def conversations_from_step( + step: Step, *, session_mode: SessionMode | None, max_messages: int +) -> list[JSONObject]: + """Convert a trace or session, preserving child order and explicit session boundaries. + + Trace containers hold content only in children. Sessions contain ordered traces. + ``per_trace`` isolates each trace; ``continuous`` allows rules across trace boundaries. + Empty containers produce no conversations. Unknown or ambiguous structures raise + StepConversionError instead of silently omitting events. + + Args: + step: Complete trace/session hierarchy in chronological order. + session_mode: Explicit boundary semantics, required for session roots. + max_messages: Maximum messages allowed in each converted conversation. + + Returns: + Quarry conversation objects, one per scan according to the session mode. + + Raises: + StepConversionError: If conversion requires guessing, dropping content, or + exceeding the configured message limit or hierarchy depth limit. + """ + if step.type not in {"trace", "session"}: + raise StepConversionError("quarry.policy requires a trace or session root Step.") + roots = [step] + if step.type == "session": + if session_mode is None: + raise StepConversionError("Session evaluation requires an explicit session_mode.") + if not _empty(step.input) or not _empty(step.output): + raise StepConversionError("Session content must be in Step.children, not input/output.") + roots = step.children or [] + if any(child.type != "trace" for child in roots): + raise StepConversionError( + "Session children must be trace steps in chronological order." + ) + builder = _ConversationBuilder(max_messages=max_messages) + if step.tools is not None: + builder.tools = step.tools + conversations: list[JSONObject] = [] + for index, root in enumerate(roots): + builder.add(root, str(index)) + if step.type == "session" and session_mode == "per_trace": + if builder.messages: + conversations.append(builder.conversation()) + builder = _ConversationBuilder(max_messages=max_messages, tools=step.tools) + if builder.messages: + conversations.append(builder.conversation()) + return conversations diff --git a/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/evaluator.py b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/evaluator.py new file mode 100644 index 00000000..3d553752 --- /dev/null +++ b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/evaluator.py @@ -0,0 +1,165 @@ +"""Local Quarry scans through the standard Agent Control evaluator interface.""" + +from __future__ import annotations + +import asyncio +from importlib.metadata import version +from typing import Any, Literal + +from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator +from agent_control_models import EvaluatorResult, Step +from pydantic import ValidationError +from quarry import ConversationValidationError, IncompleteScanError, SafetyLimits, compile_policy +from quarry.config import QuarryConfig + +from .config import QuarryEvaluatorConfig +from .conversion import StepConversionError, conversations_from_step + + +@register_evaluator +class QuarryEvaluator(Evaluator[QuarryEvaluatorConfig]): + """Compile one local rule per cached instance; keep all scan state request-local.""" + + metadata = EvaluatorMetadata( + name="quarry.policy", + version=version("agent-control-evaluator-quarry"), + description="Local symbolic Quarry policy evaluation over trace/session steps", + requires_api_key=False, + timeout_ms=10000, + ) + config_model = QuarryEvaluatorConfig + + def __init__(self, config: QuarryEvaluatorConfig) -> None: + """Compile before accepting scans; reject invalid rules and missing resources. + + No models, adapters, text files, or stub probers are configured. Rules requiring + these resources fail at construction. Agent Control's evaluator factory caches + the instance by configuration, including the rule source and limits. + + Args: + config: Validated rule source, limits, and session semantics. + + Raises: + ValueError: If the rule cannot compile as a local symbolic policy. + """ + super().__init__(config) + runtime = QuarryConfig( + config_version=2, + safety_limits=SafetyLimits( + rule_timeout=self.get_timeout_seconds(), + max_messages=config.max_messages, + enforce_limits=True, + ), + ) + try: + self._policy = compile_policy(config.rule, runtime_config=runtime) + if self._policy.parameter_schema: + raise ValueError("Parameterized rules are not supported by this integration.") + except Exception as exc: + raise ValueError( + "Quarry rule could not compile. Supply exactly one valid symbolic rule " + "without parameters, model, text-file, or prober dependencies." + ) from exc + + async def evaluate(self, data: Any) -> EvaluatorResult: + """Evaluate a complete Step or its serialized object. + + Args: + data: Trace/session Step or its serialized object; empty input is a non-match. + + Returns: + A policy decision, or an error for invalid or incomplete evaluation. + """ + if data is None or (isinstance(data, (str, list, dict)) and not data): + return self._empty_result() + try: + step = data if isinstance(data, Step) else Step.model_validate(data) + except ValidationError: + return self._error("A complete trace/session Step is required.", "invalid") + return await self._evaluate_step(step) + + async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: + """Scan the complete runtime Step, independent of selector-selected scalar data. + + Controls should use selector path ``*`` and scope to trace/session. The complete + hierarchy is authoritative; a missing selected field does not suppress its scan. + + Args: + data: Selector-selected value; the complete step takes precedence. + step: Authoritative trace/session hierarchy for this request. + + Returns: + A policy decision, or an error for invalid or incomplete evaluation. + """ + return await self._evaluate_step(step) + + async def _evaluate_step(self, step: Step) -> EvaluatorResult: + loop = asyncio.get_running_loop() + deadline = loop.time() + self.get_timeout_seconds() + try: + conversations = conversations_from_step( + step, session_mode=self.config.session_mode, max_messages=self.config.max_messages + ) + if loop.time() >= deadline: + raise TimeoutError + if not conversations: + return self._empty_result() + matches: set[str] = set() + # All traces must finish authoritatively, even after an earlier trace matched. + async with asyncio.timeout_at(deadline): + for conversation in conversations: + remaining = deadline - loop.time() + if remaining <= 0: + raise TimeoutError + result = await self._policy.scan_async( + conversation, deadline_seconds=remaining + ) + result.raise_for_status() + if loop.time() >= deadline: + raise TimeoutError + matches.update(result.rule_matches) + return EvaluatorResult( + matched=bool(matches), + confidence=1.0, + message="Quarry policy matched." if matches else "Quarry policy did not match.", + metadata={ + "completion_state": "complete", + "matched_rules": sorted(matches), + "scan_count": len(conversations), + "session_mode": self.config.session_mode if step.type == "session" else None, + "policy_digest": self._policy.policy_digest, + }, + ) + except StepConversionError as exc: + return self._error(str(exc), "invalid") + except ConversationValidationError: + return self._error("Quarry rejected an invalid conversation.", "invalid") + except IncompleteScanError: + return self._error( + "Quarry scan incomplete: a rule failed, timed out, was skipped, or hit a limit.", + "incomplete", + ) + except TimeoutError: + return self._error("Quarry evaluation timed out.", "incomplete") + except Exception: + # Upstream diagnostics can contain conversation data. Do not expose them. + return self._error("Quarry evaluation failed.", "incomplete") + + @staticmethod + def _empty_result() -> EvaluatorResult: + return EvaluatorResult( + matched=False, + confidence=1.0, + message="No conversation messages to scan.", + metadata={"completion_state": "complete", "matched_rules": [], "scan_count": 0}, + ) + + @staticmethod + def _error(message: str, state: Literal["invalid", "incomplete"]) -> EvaluatorResult: + return EvaluatorResult( + matched=False, + confidence=0.0, + message=message, + error=message, + metadata={"completion_state": state}, + ) diff --git a/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/py.typed b/evaluators/contrib/quarry/src/agent_control_evaluator_quarry/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/evaluators/contrib/quarry/tests/test_quarry.py b/evaluators/contrib/quarry/tests/test_quarry.py new file mode 100644 index 00000000..935553d1 --- /dev/null +++ b/evaluators/contrib/quarry/tests/test_quarry.py @@ -0,0 +1,581 @@ +"""Behavioral coverage using the installed Quarry compiler and scanner.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from importlib.metadata import entry_points +from unittest.mock import AsyncMock, patch + +import pytest +from agent_control_engine.core import ControlEngine +from agent_control_evaluators import clear_evaluator_cache, get_evaluator_instance +from agent_control_models import ( + ControlDefinition, + EvaluationRequest, + EvaluatorSpec, + JSONObject, + JSONValue, + Step, +) +from pydantic import ValidationError +from quarry import CompiledPolicy, ScanResult, compile_policy + +from agent_control_evaluator_quarry import QuarryEvaluator, QuarryEvaluatorConfig + +RULE = 'rule "Blocked request"\ncondition:\n user_said("blocked")\n' +SEQUENCE = 'rule "Read then send"\ncondition:\n has_sequence("read", "send")\n' + + +def trace(*children: Step) -> Step: + return Step(type="trace", name="trace", input=None, children=list(children)) + + +def turn(text: str, output: JSONValue = None) -> Step: + return Step(type="llm", name="model", input=text, output=output) + + +def tool(name: str, *, call_id: str | None = None) -> Step: + return Step( + type="tool", + name=name, + input={"query": "value"}, + output="done", + context={"tool_call_id": call_id} if call_id else None, + ) + + +def evaluator(**config: JSONValue) -> QuarryEvaluator: + return QuarryEvaluator(QuarryEvaluatorConfig.model_validate({"rule": RULE, **config})) + + +@pytest.mark.parametrize("data", [None, "", [], {}]) +@pytest.mark.asyncio +async def test_empty_input_is_nonmatch(data: JSONValue) -> None: + # Given: a valid local policy and empty selected data + guard = evaluator() + # When: evaluating empty input + result = await guard.evaluate(data) + # Then: empty data is not an infrastructure failure + assert result.matched is False + assert result.error is None + + +@pytest.mark.parametrize("text,matched", [("blocked operation", True), ("hello", False)]) +@pytest.mark.asyncio +async def test_real_policy_matches(text: str, matched: bool) -> None: + # Given: an ordinary trace and the real Quarry compiler + guard = evaluator() + # When: evaluating the recursive Step public contract + result = await guard.evaluate(trace(turn(text)).model_dump(mode="json")) + # Then: a policy match is a normal result, with rule identity and complete status + assert result.error is None + assert result.matched is matched + assert result.metadata is not None + assert result.metadata["matched_rules"] == (["Blocked request"] if matched else []) + assert result.metadata["completion_state"] == "complete" + + +@pytest.mark.asyncio +async def test_empty_hierarchy_is_nonmatch() -> None: + # Given: nested containers without messages + step = trace(trace()) + # When: evaluating the hierarchy + result = await evaluator().evaluate(step) + # Then: the evaluator does not invent messages from container names + assert result.matched is False + assert result.error is None + + +@pytest.mark.parametrize( + "config", + [ + {"rule": " "}, + {"timeout_ms": 0}, + {"timeout_ms": True}, + {"max_messages": 0}, + {"session_mode": "automatic"}, + {"model": "remote"}, + ], +) +def test_invalid_config_is_rejected(config: dict[str, object]) -> None: + # Given: an unsupported or unsafe configuration + # When: validating through the public config model + with pytest.raises(ValidationError): + QuarryEvaluatorConfig.model_validate({"rule": RULE, **config}) + # Then: no policy instance was admitted + + +@pytest.mark.parametrize( + "source", + [ + "not a rule", + RULE + RULE.replace("Blocked request", "Second"), + 'rule "Remote"\ncondition:\n model("guard").complete(messages).content contains "x"\n', + 'rule "Missing text"\ncondition:\n user_said(text_ref("secret"))\n', + 'rule "Parameters"\nparameters:\n threshold: number\n' + 'condition:\n 1 > params.threshold\n', + ], +) +def test_unusable_policy_fails_at_construction(source: str) -> None: + # Given: invalid source, multiple rules, or a rule needing unavailable resources + # When: constructing the evaluator through its public factory + with pytest.raises(ValueError, match="could not compile"): + QuarryEvaluator.from_dict({"rule": source}) + # Then: no incomplete rule generation can serve requests + + +@pytest.mark.asyncio +async def test_nested_tools_keep_sequence_and_arguments() -> None: + # Given: nested trace children whose tool arguments and order matter + source = """rule "Ordered arguments" +condition: + has_sequence("read", "send") and + any call in tool_calls where call.name == "send" and call.arguments.query == "value" +""" + step = trace(trace(tool("read")), tool("send")) + # When: scanning through the real Quarry runtime + result = await evaluator(rule=source).evaluate(step) + # Then: the conversion preserved event order, tool names, and structured arguments + assert result.error is None + assert result.matched is True + + +@pytest.mark.asyncio +async def test_tool_children_and_structured_result_keep_order() -> None: + # Given: a parent tool whose child runs before the parent returns structured output + parent = Step( + type="tool", name="parent", input={}, output={"status": "blocked"}, children=[tool("read")] + ) + source = """rule "Nested tool result" +condition: + has_sequence("parent", "read") and + any response in get_tool_responses("parent") where response.content contains "blocked" +""" + # When: evaluating the nested tool hierarchy + result = await evaluator(rule=source).evaluate(trace(parent)) + # Then: nested tool calls are distinct and the structured parent result remains visible + assert result.error is None + assert result.matched is True + + +@pytest.mark.asyncio +async def test_offered_tools_are_not_executed_calls() -> None: + # Given: a model offered a tool without calling it + step = trace( + Step( + type="llm", + name="model", + input="hello", + tools=[ + { + "type": "function", + "function": {"name": "read", "parameters": {"type": "object"}}, + } + ], + ) + ) + source = 'rule "Offered only"\ncondition:\n has_tool("read") and not called_tool("read")\n' + # When: scanning the real Quarry tools surface + result = await evaluator(rule=source).evaluate(step) + # Then: tool availability stays separate from events that actually occurred + assert result.error is None + assert result.matched is True + + +@pytest.mark.asyncio +async def test_pending_tool_call_has_no_invented_result() -> None: + # Given: a tool pre-check with no result yet + step = trace(Step(type="tool", name="read", input={"query": "value"})) + source = '''rule "Pending call" +condition: + called_tool("read") and + not (any response in tool_responses where response.content contains "done") +''' + # When: evaluating the trace before the tool returns + result = await evaluator(rule=source).evaluate(step) + # Then: the call is visible but no response was invented + assert result.error is None + assert result.matched is True + + +@pytest.mark.parametrize("mode,matched,scans", [("per_trace", False, 2), ("continuous", True, 1)]) +@pytest.mark.asyncio +async def test_session_boundaries_are_explicit(mode: str, matched: bool, scans: int) -> None: + # Given: a sequence split across two ordered traces + step = Step( + type="session", + name="session", + input=None, + children=[trace(tool("read")), trace(tool("send"))], + ) + # When: scanning with the caller's explicit boundary choice + result = await evaluator(rule=SEQUENCE, session_mode=mode).evaluate(step) + # Then: only continuous sessions permit rules to span traces + assert result.error is None + assert result.matched is matched + assert result.metadata is not None + assert result.metadata["scan_count"] == scans + assert result.metadata["session_mode"] == mode + + +@pytest.mark.parametrize( + "step,reason", + [ + (Step(type="session", name="s", input=None, children=[]), "session_mode"), + ( + Step(type="session", name="s", input=None, children=[turn("blocked")]), + "Session children", + ), + (trace(Step(type="retriever", name="r", input="blocked")), "Supported child types"), + (turn("blocked"), "trace or session root"), + (Step(type="trace", name="t", input="blocked"), "Step.children"), + (trace(Step(type="llm", name="m", input={"prompt": "blocked"})), "messages array"), + (trace(Step(type="llm", name="m", input=None, output={"value": "blocked"})), "LLM output"), + ( + trace(Step(type="llm", name="m", input="blocked", children=[tool("read")])), + "cannot have", + ), + ], +) +@pytest.mark.asyncio +async def test_ambiguous_steps_report_error(step: Step, reason: str) -> None: + # Given: a shape that would require guessing or dropping content + guard = evaluator(session_mode=None if not step.children else "per_trace") + # When: attempting a scan + result = await guard.evaluate(step) + # Then: unsupported data is an explicit evaluation error, not a clean pass + assert result.matched is False + assert result.error is not None and reason in result.error + + +@pytest.mark.asyncio +async def test_cumulative_history_and_tool_step_do_not_duplicate_calls() -> None: + # Given: an assistant call, a linked tool execution, and a later full LLM history + call: JSONObject = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "read", "arguments": '{"query":"value"}'}, + } + ], + } + history: list[JSONValue] = [ + {"role": "user", "content": "hello"}, + call, + {"role": "tool", "tool_call_id": "call-1", "content": "done"}, + ] + source = 'rule "Single call"\ncondition:\n count_tool_calls("read") == 1\n' + step = trace( + turn("hello", call), + tool("read", call_id="call-1"), + Step(type="llm", name="next", input={"messages": history}, output="answer"), + ) + before = step.model_dump(mode="json") + # When: scanning the linked recursive representation + result = await evaluator(rule=source).evaluate(step) + # Then: the call exists exactly once and the supplied Step is unchanged + assert result.error is None + assert result.matched is True + assert step.model_dump(mode="json") == before + + +@pytest.mark.parametrize( + "call_id,arguments,name,reason", + [ + (None, '{"query":"value"}', "read", "need context.tool_call_id"), + ("call-1", '{"query":"different"}', "read", "arguments do not match"), + ("call-1", '{"query":"value"}', "other", "name does not match"), + ], +) +@pytest.mark.asyncio +async def test_tool_link_requires_consistent_identity( + call_id: str | None, arguments: str, name: str, reason: str +) -> None: + # Given: a tool step that cannot reliably be linked to its assistant call + call: JSONObject = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": name, "arguments": arguments}, + } + ], + } + # When: evaluating the trace + result = await evaluator().evaluate(trace(turn("hello", call), tool("read", call_id=call_id))) + # Then: identity mismatches cannot silently invent a second call + assert result.error is not None and reason in result.error + + +@pytest.mark.asyncio +async def test_invalid_quarry_messages_return_safe_error() -> None: + # Given: an orphan tool response with sensitive data in its identifier + step = trace( + Step( + type="llm", + name="m", + input=[ + { + "role": "tool", + "tool_call_id": "private-secret-id", + "content": "secret", + } + ], + ) + ) + # When: Quarry validates the conversation + result = await evaluator().evaluate(step) + # Then: the scan fails without returning upstream input-bearing diagnostics + assert result.matched is False + assert result.error is not None and "invalid conversation" in result.error + assert "private-secret-id" not in result.model_dump_json() + + +@pytest.mark.asyncio +async def test_bad_tool_json_is_rejected_by_quarry() -> None: + # Given: malformed tool arguments and a rule that reads their structured fields + call: JSONObject = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "read", "arguments": "{broken-private-value"}, + } + ], + } + source = """rule "Inspect arguments" +condition: + any call in tool_calls where call.arguments.query == "value" +""" + # When: the real pinned Quarry runtime validates those arguments + result = await evaluator(rule=source).evaluate(trace(turn("hello", call))) + # Then: a decode failure is not mistaken for a clean non-match + assert result.matched is False + assert result.error is not None and "invalid conversation" in result.error + assert "private-value" not in result.model_dump_json() + + +@pytest.mark.asyncio +async def test_divergent_history_is_rejected() -> None: + # Given: a later history that would rewrite a previously emitted message + step = trace( + turn("blocked"), + Step( + type="llm", + name="next", + input=[ + { + "role": "user", + "content": "different", + } + ], + ), + ) + # When: converting cumulative histories + result = await evaluator().evaluate(step) + # Then: the evaluator cannot silently replace or duplicate the conversation + assert result.matched is False + assert result.error is not None and "exact conversation prefix" in result.error + + +@pytest.mark.asyncio +async def test_deep_hierarchy_is_bounded() -> None: + # Given: more container levels than the documented conversion limit + step = turn("blocked") + for _ in range(34): + step = trace(step) + # When: evaluating the recursive hierarchy + result = await evaluator().evaluate(step) + # Then: the evaluator fails explicitly without omitting deeper events + assert result.matched is False + assert result.error is not None and "32 levels" in result.error + + +@pytest.mark.parametrize( + "scan", + [ + ScanResult( + file_path=None, + rule_matches=["Blocked request"], + scan_time=0, + timed_out_rules=["Blocked request"], + ), + ScanResult(file_path=None, rule_matches=[], scan_time=0, skipped_rules=["Blocked request"]), + ScanResult(file_path=None, rule_matches=[], scan_time=0, was_truncated=True), + ScanResult( + file_path=None, + rule_matches=[], + scan_time=0, + evaluation_errors={"Blocked request": "secret"}, + ), + ScanResult(file_path=None, rule_matches=[], scan_time=0, evaluation_warnings=["limited"]), + ], +) +@pytest.mark.asyncio +async def test_incomplete_scans_never_return_partial_decisions(scan: ScanResult) -> None: + # Given: the upstream scan boundary is mocked to force rare incomplete states + guard = evaluator() + with patch.object(CompiledPolicy, "scan_async", new=AsyncMock(return_value=scan)): + # When: scanning through the public evaluator + result = await guard.evaluate(trace(turn("blocked"))) + # Then: neither partial matches nor partial clean results are authoritative + assert result.matched is False + assert result.error is not None and "incomplete" in result.error + assert "secret" not in result.model_dump_json() + + +@pytest.mark.asyncio +async def test_infrastructure_failure_is_error() -> None: + # Given: the upstream boundary is mocked to simulate an unexpected runtime failure + guard = evaluator() + with patch.object( + CompiledPolicy, "scan_async", new=AsyncMock(side_effect=RuntimeError("secret")) + ): + # When: scanning the complete trace + result = await guard.evaluate(trace(turn("blocked"))) + # Then: infrastructure failure is visible without leaking raw diagnostics + assert result.matched is False + assert result.error == "Quarry evaluation failed." + + +@pytest.mark.asyncio +async def test_timeout_returns_error() -> None: + # Given: the upstream boundary waits indefinitely so only our deadline can end it + guard = evaluator(timeout_ms=10) + + async def wait_for_scan(*args: object, **kwargs: object) -> ScanResult: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + with patch.object(CompiledPolicy, "scan_async", new=wait_for_scan): + # When: scanning with a bounded timeout + result = await guard.evaluate(trace(turn("blocked"))) + # Then: timeout is an evaluation error rather than a policy decision + assert result.matched is False + assert result.error == "Quarry evaluation timed out." + + +@pytest.mark.asyncio +async def test_cancellation_propagates() -> None: + # Given: upstream cancellation is injected at the scan boundary + guard = evaluator() + with patch.object( + CompiledPolicy, "scan_async", new=AsyncMock(side_effect=asyncio.CancelledError) + ): + # When: the caller cancels evaluation + with pytest.raises(asyncio.CancelledError): + await guard.evaluate(trace(turn("blocked"))) + # Then: cached evaluation does not swallow cancellation + + +@pytest.mark.asyncio +async def test_later_trace_error_overrides_earlier_match() -> None: + # Given: a session whose first trace matches and second trace is invalid + invalid = Step(type="llm", name="m", input=[{"role": "unsupported", "content": "hello"}]) + session = Step( + type="session", name="s", input=None, children=[trace(turn("blocked")), trace(invalid)] + ) + # When: all independent traces are scanned + result = await evaluator(session_mode="per_trace").evaluate(session) + # Then: an early match cannot hide incomplete session coverage + assert result.matched is False + assert result.error is not None + + +@pytest.mark.asyncio +async def test_message_limit_is_error_without_truncation() -> None: + # Given: a trace longer than the configured scan limit + guard = evaluator(max_messages=1) + # When: converting the full trace + result = await guard.evaluate(trace(turn("blocked", "answer"))) + # Then: no partial match is reported for a truncated conversation + assert result.matched is False + assert result.error is not None and "max_messages" in result.error + + +@pytest.mark.asyncio +async def test_cached_policy_is_reused_and_requests_are_isolated() -> None: + # Given: the compile I/O boundary is wrapped to observe the public factory cache + clear_evaluator_cache() + spec = EvaluatorSpec(name="quarry.policy", config={"rule": RULE}) + with patch( + "agent_control_evaluator_quarry.evaluator.compile_policy", wraps=compile_policy + ) as compile_mock: + first = get_evaluator_instance(spec) + second = get_evaluator_instance(spec) + # When: concurrent requests use the same cached instance + results = await asyncio.gather( + first.evaluate(trace(turn("blocked"))), + second.evaluate(trace(turn("hello"))), + ) + # Then: compilation happens once and conversation state does not cross requests + assert first is second + assert compile_mock.call_count == 1 + assert [result.matched for result in results] == [True, False] + assert all(result.error is None for result in results) + clear_evaluator_cache() + + +def test_installed_entry_point_loads_evaluator() -> None: + # Given: the optional distribution is installed + entries = entry_points(group="agent_control.evaluators", name="quarry.policy") + # When: discovery loads the published entry point + loaded = next(iter(entries)).load() + # Then: the entry point and evaluator metadata agree + assert loaded is QuarryEvaluator + assert loaded.metadata.name == "quarry.policy" + + +@dataclass +class _Control: + id: int + name: str + control: ControlDefinition + + +@pytest.mark.parametrize("step_type", ["trace", "session"]) +@pytest.mark.asyncio +async def test_engine_scopes_and_context_hook(step_type: str) -> None: + # Given: a trace/session control and a selector with no selected content + control = ControlDefinition.model_validate( + { + "enabled": True, + "execution": "server", + "description": "Quarry integration", + "scope": {"step_types": [step_type], "stages": ["post"]}, + "condition": { + "selector": {"path": "context.absent"}, + "evaluator": { + "name": "quarry.policy", + "config": { + "rule": RULE, + "session_mode": "continuous", + }, + }, + }, + "action": {"decision": "deny"}, + } + ) + engine = ControlEngine([_Control(1, "quarry-test", control)]) + step = trace(turn("blocked")) + if step_type == "session": + step = Step(type="session", name="s", input=None, children=[step]) + # When: the existing engine evaluates the configured scope and context hook + result = await engine.process( + EvaluationRequest( + agent_name="test-agent", + step=step, + stage="post", + ) + ) + # Then: complete Step data triggers the control through the standard engine + assert result.is_safe is False + assert result.matches[0].result.error is None diff --git a/pyproject.toml b/pyproject.toml index c12c819f..c1d06c2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,7 @@ version_toml = [ "evaluators/contrib/cisco/pyproject.toml:project.version", "evaluators/contrib/defenseclaw/pyproject.toml:project.version", "evaluators/contrib/galileo/pyproject.toml:project.version", + "evaluators/contrib/quarry/pyproject.toml:project.version", ] version_source = "tag" commit_message = "chore(release): v{version}" diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 3728a5f8..4efe0304 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] [project.urls] diff --git a/server/tests/test_auth_framework.py b/server/tests/test_auth_framework.py index 8b3d0e6c..3cc541a4 100644 --- a/server/tests/test_auth_framework.py +++ b/server/tests/test_auth_framework.py @@ -1016,15 +1016,6 @@ async def test_http_upstream_4xx_diagnostics_redact_dynamic_location_and_type( "parsed", id="entry-not-an-object", ), - pytest.param( - lambda req: httpx.Response( - 422, - content=b'{"detail":' + b"[" * 10_000 + b"0" + b"]" * 10_000 + b"}", - ), - "unknown", - "unusable", - id="deeply-nested-json", - ), pytest.param( lambda req: httpx.Response( 422, @@ -1066,6 +1057,32 @@ async def test_http_upstream_4xx_diagnostics_tolerate_unexpected_bodies( assert f"upstream_validation_status={expected_status}" in record.getMessage() +@pytest.mark.asyncio +async def test_http_upstream_4xx_diagnostics_tolerate_deeply_nested_json( + caplog: pytest.LogCaptureFixture, +) -> None: + # Given: a valid JSON rejection body beyond older interpreters' decoder recursion limit + provider = _build_upstream( + lambda req: httpx.Response( + 422, content=b'{"detail":' + b"[" * 10_000 + b"0" + b"]" * 10_000 + b"}", + ) + ) + + # When: authorization receives the rejection + with caplog.at_level(logging.WARNING): + with pytest.raises(APIError) as exc_info: + await provider.authorize(_build_request(), Operation.CONTROL_BINDINGS_WRITE) + + # Then: decoder depth limits affect only the diagnostic count, never the 502 decision + record = _rejection_record(caplog) + assert exc_info.value.status_code == 502 + assert record.__dict__["upstream_validation"] == {} + assert ( + record.__dict__["upstream_validation_total"], + record.__dict__["upstream_validation_status"], + ) in ((1, "parsed"), ("unknown", "unusable")) + + @pytest.mark.asyncio async def test_http_upstream_4xx_diagnostics_distinguish_empty_validation_list( caplog: pytest.LogCaptureFixture,