From 99a4779f9c7aa45a93802635418856808e4e446d Mon Sep 17 00:00:00 2001 From: "openrouter-port-bot[bot]" <312657780+openrouter-port-bot[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:52:12 +0000 Subject: [PATCH] port: sync with @openrouter/agent upstream --- .upstreamer/eval-report.md | 650 +++--- README.md | 27 +- src/openrouter_agent/__init__.py | 136 ++ src/openrouter_agent/anthropic_compat.py | 9 +- src/openrouter_agent/async_params.py | 6 + src/openrouter_agent/call_model.py | 25 +- src/openrouter_agent/chat_compat.py | 51 +- src/openrouter_agent/claude_type_guards.py | 78 +- src/openrouter_agent/conversation_state.py | 67 +- src/openrouter_agent/doom_loop.py | 1289 ++++++++++++ src/openrouter_agent/hooks_schemas.py | 48 +- src/openrouter_agent/hooks_types.py | 6 +- src/openrouter_agent/item_types.py | 9 + src/openrouter_agent/model_result.py | 1825 ++++++++++++++--- src/openrouter_agent/next_turn_params.py | 80 +- src/openrouter_agent/openrouter.py | 36 +- src/openrouter_agent/reusable_stream.py | 394 +++- src/openrouter_agent/stop_conditions.py | 16 +- src/openrouter_agent/stream_transformers.py | 33 + src/openrouter_agent/tool.py | 50 +- .../tool_event_broadcaster.py | 100 +- src/openrouter_agent/tool_executor.py | 48 +- src/openrouter_agent/tool_set.py | 581 ++++++ src/openrouter_agent/tool_types.py | 19 +- src/openrouter_agent/turn_context.py | 21 + src/openrouter_agent/user_agent.py | 68 + tests/unit/test_approval_gate_regressions.py | 1630 +++++++++++++++ tests/unit/test_call_model.py | 5 +- tests/unit/test_call_model_active_tools.py | 183 ++ tests/unit/test_chat_compat.py | 191 ++ tests/unit/test_claude_type_guards.py | 106 + .../test_conversation_state_serialization.py | 42 + tests/unit/test_doom_loop.py | 926 +++++++++ tests/unit/test_doom_loop_escalation.py | 513 +++++ tests/unit/test_doom_loop_fanout.py | 461 +++++ tests/unit/test_doom_loop_integration.py | 793 +++++++ tests/unit/test_doom_loop_public_api.py | 134 ++ tests/unit/test_doom_loop_remediation.py | 983 +++++++++ tests/unit/test_final_response_validation.py | 176 ++ tests/unit/test_get_usage.py | 374 ++++ tests/unit/test_hitl_tool.py | 889 ++++++++ tests/unit/test_hooks_post_model_call.py | 68 + tests/unit/test_hooks_session_lifecycle.py | 69 + tests/unit/test_manual_tool_wire_schema.py | 168 ++ .../unit/test_max_output_tokens_truncation.py | 129 ++ tests/unit/test_mcp_tool_branding.py | 29 + tests/unit/test_model_result_hooks.py | 4 +- tests/unit/test_next_turn_params.py | 368 ++++ tests/unit/test_next_turn_tool_choice_loop.py | 110 + tests/unit/test_openrouter.py | 192 ++ tests/unit/test_replay_buffer_compaction.py | 125 ++ tests/unit/test_reusable_stream.py | 257 +++ tests/unit/test_server_tool.py | 171 ++ .../test_state_context_and_broadcaster.py | 5 +- tests/unit/test_stop_conditions.py | 174 ++ tests/unit/test_stop_when_before_approval.py | 124 ++ tests/unit/test_tool_choice_relaxation.py | 365 ++++ tests/unit/test_tool_context.py | 251 +++ tests/unit/test_tool_event_broadcaster.py | 354 ++++ tests/unit/test_tool_executor_return.py | 232 +++ tests/unit/test_tool_factory_and_executor.py | 4 +- tests/unit/test_tool_name_events.py | 233 +++ tests/unit/test_tool_orchestrator.py | 512 +++++ tests/unit/test_tool_set.py | 616 ++++++ tests/unit/test_tool_terminal_empty_final.py | 17 +- tests/unit/test_turn_context.py | 96 + tests/unit/test_turn_end_race_condition.py | 6 +- tests/unit/test_user_agent.py | 29 + tests/vectors/doom-loop-fingerprints.json | 132 ++ upstreamer-changelog.md | 59 + 70 files changed, 17288 insertions(+), 689 deletions(-) create mode 100644 src/openrouter_agent/doom_loop.py create mode 100644 src/openrouter_agent/tool_set.py create mode 100644 src/openrouter_agent/user_agent.py create mode 100644 tests/unit/test_approval_gate_regressions.py create mode 100644 tests/unit/test_call_model_active_tools.py create mode 100644 tests/unit/test_chat_compat.py create mode 100644 tests/unit/test_claude_type_guards.py create mode 100644 tests/unit/test_doom_loop.py create mode 100644 tests/unit/test_doom_loop_escalation.py create mode 100644 tests/unit/test_doom_loop_fanout.py create mode 100644 tests/unit/test_doom_loop_integration.py create mode 100644 tests/unit/test_doom_loop_public_api.py create mode 100644 tests/unit/test_doom_loop_remediation.py create mode 100644 tests/unit/test_final_response_validation.py create mode 100644 tests/unit/test_get_usage.py create mode 100644 tests/unit/test_hitl_tool.py create mode 100644 tests/unit/test_hooks_post_model_call.py create mode 100644 tests/unit/test_hooks_session_lifecycle.py create mode 100644 tests/unit/test_manual_tool_wire_schema.py create mode 100644 tests/unit/test_max_output_tokens_truncation.py create mode 100644 tests/unit/test_next_turn_params.py create mode 100644 tests/unit/test_next_turn_tool_choice_loop.py create mode 100644 tests/unit/test_openrouter.py create mode 100644 tests/unit/test_replay_buffer_compaction.py create mode 100644 tests/unit/test_reusable_stream.py create mode 100644 tests/unit/test_server_tool.py create mode 100644 tests/unit/test_stop_conditions.py create mode 100644 tests/unit/test_stop_when_before_approval.py create mode 100644 tests/unit/test_tool_choice_relaxation.py create mode 100644 tests/unit/test_tool_context.py create mode 100644 tests/unit/test_tool_event_broadcaster.py create mode 100644 tests/unit/test_tool_executor_return.py create mode 100644 tests/unit/test_tool_name_events.py create mode 100644 tests/unit/test_tool_orchestrator.py create mode 100644 tests/unit/test_tool_set.py create mode 100644 tests/unit/test_turn_context.py create mode 100644 tests/vectors/doom-loop-fingerprints.json diff --git a/.upstreamer/eval-report.md b/.upstreamer/eval-report.md index 95e04b6..86cb603 100644 --- a/.upstreamer/eval-report.md +++ b/.upstreamer/eval-report.md @@ -1,267 +1,383 @@ -PASS WITH WARNINGS - -# Port Parity Eval — Python (`@openrouter/agent` 0.7.2 → 0.8.0), second pass - -Fresh re-review against upstream commit `680bceb4598f228d3e2ec58e2416e4335cdff059`, -delta baseline `adc7939f4b7ed85b1a060d13433b8be6063cff73`, scoped to `packages/agent/`. -This is a second-pass review after a prior FAIL with three findings. All three -are verified fixed below by reading the code directly (not by trusting the -converter's summary), and a broader sweep of the rest of the delta and the -eval's Required Qualities turned up no new material gap — only pre-existing, -cosmetic items. - -## Verdict rationale - -All three prior findings are genuinely fixed, confirmed by tracing execution -and by breaking the fix and watching the test fail. No required public API -symbol is missing, no load-bearing loop / state / approval-ordering / hooks -behavior regressed, and the declared version (`0.8.0`) matches what's -actually ported. Remaining items are documentation/coverage gaps that don't -change behavior — hence WARNINGS, not FAIL. - -## Finding 1 (prior FAIL #1) — Stop hook `force_resume` zero-cost retry: FIXED, verified - -`src/openrouter_agent/model_result.py:677-694`, inside `_run()`: - -```python -while stop_conditions and await is_stop_condition_met(stop_conditions, self._steps): - stop_decision = await self._run_stop_hook(force_resume_count, current_request) - if stop_decision == "resume": - force_resume_count += 1 - continue - session_end_reason = "max_turns" - stopped_by_stop_when = True - break -``` - -- `is_stop_condition_met` (`stop_conditions.py`) is a pure function over - `self._steps` (already-collected `StepResult`s) — it does not call `_send` - or `_send_and_track`. The `continue` re-enters the `while` and re-evaluates - this pure check against the *same* steps; no request is sent on the - "resume" path. This is a faithful match of upstream's - `model-result.ts:3050-3068` (`shouldStopExecution()` / `runStopHook()` / - `continue`), confirmed by reading both side by side. -- `tests/unit/test_model_result_hooks.py::test_stop_hook_force_resume_is_a_zero_cost_retry_no_extra_model_request` - genuinely pins this: the mock `QueuedResponses` has exactly one queued - response, and asserts `len(client.beta.responses.requests) == 1` after a - run in which the Stop hook is invoked twice (once forcing resume, once - not). If the "resume" branch sent a real request, the mock's second - `pop(0)` would raise `IndexError` on the empty list. -- I verified this is a real regression guard, not an accidental pass: I - temporarily replaced the `continue` branch with a version that calls - `await self._send_and_track(...)` (simulating the old bug), reran the test, - and both `test_stop_hook_force_resume_is_a_zero_cost_retry_no_extra_model_request` - and `test_stop_hook_force_resume_then_falls_through_to_normal_tool_round` - failed immediately with `IndexError: pop from empty list`. Reverted - afterward; `git diff --stat` confirms the file is back to its pre-sabotage - state and the tests pass again. -- Minor cosmetic nit: the second test's docstring says the halted round's - tool calls "execute via the normal round path (not the final-directive - coercion)" — tracing the code, they actually go through the *same* - final-directive-coercion branch (`stopped_by_stop_when` → `resolvable_pending` - execute → `tool_choice: "none"` follow-up), which is also what upstream does - (`model-result.ts:3205+`). The assertions themselves (text output, request - count) are correct; only the prose comment is a little misleading. Not a - behavioral gap. - -## Finding 2 (prior FAIL #2) — MCP tool-branding surface: FIXED, verified - -- `tool.py:57-64` — `mark_mcp()` returns `{**tool_to_mark, "_mcp": True}`, a - new dict; the underlying `"function"` value is not copied (shallow), same - as upstream's `{...toolToMark, _mcp: true}`. Confirmed non-mutating via - `tests/unit/test_mcp_tool_branding.py::test_mark_mcp_is_non_mutating_and_is_mcp_tool_detects_the_brand` - (`base is not branded`, `branded["function"] is base["function"]`). -- `tool_types.py:264-270` — `McpBranded` alias and `is_mcp_tool()` structural - check (`tool.get("_mcp") is True`), matching upstream's `isMcpTool`. -- `tool_executor.py` — `source = "mcp" if is_mcp_tool(tool) else "client"` is - computed and threaded through all three execute paths - (`execute_regular_tool`, `execute_generator_tool`, `execute_hitl_tool`, - lines 62/86/151) and returned in every result dict (success and error - branches). -- `model_result.py:825, 852` — the `tool.result` / `tool_result` stream event - carries `"source"`, preferring the executor-computed `result.get("source", ...)` - and falling back to `is_mcp_tool(tool)` for the parse-error path where no - executor result exists. -- `test_mcp_tool_branding.py::test_mcp_branded_tool_result_carries_source_mcp_in_tool_result_event` - and `test_regular_client_tool_result_carries_source_client` both drive - `call_model(...)` end-to-end through a mocked Responses API and assert - `source` on the actual `tool_result` event from `get_tool_stream()` — this - is genuine parity coverage, not a unit-level stub check. - -## Finding 3 (prior FAIL #3) — Over-exported hook internals: FIXED, verified - -- `src/openrouter_agent/__init__.py` — grepped the import block and `__all__`: - `BUILT_IN_HOOKS`, `BUILT_IN_HOOK_NAMES`, `matches_tool`, `resolve_hooks`, - `execute_handler_chain` appear nowhere in either. -- No wildcard (`import *`) re-export exists anywhere in `__init__.py` that - could smuggle these back in. `resolve_hooks` is used internally by - `call_model.py` via `from .hooks_resolve import resolve_hooks` but is never - re-exported from the package `__init__`. -- `.upstreamer/scripts/verify.sh`'s `REQUIRED_SYMBOLS` list was inspected — - it does not reference any of the five removed names, so the mechanical - verifier's required-symbol check is unaffected by the removal. -- This matches upstream's own `index.ts`, which explicitly documents (in a - code comment added in this exact delta) that `matchesTool`, `resolveHooks`, - `BUILT_IN_HOOKS`, and raw schema objects are deliberately not exported. - -## Finding 4 (prior FAIL #4) — Thin hook test coverage: FIXED, verified - -Read all four new tests in `tests/unit/test_model_result_hooks.py` in full: - -- `test_user_prompt_submit_can_reject_a_string_prompt` — asserts the raised - `ValueError` message and, critically, `len(client.beta.responses.requests) == 0`, - proving the model was never called after rejection (not just that an - exception was raised somewhere). -- `test_user_prompt_submit_can_mutate_a_string_prompt` — asserts the - *actually-sent* request's `input[0]["content"]` contains the mutated text, - reading `client.beta.responses.requests[0]`, i.e., proving the mutation - reached the wire request, not just the hook's return value. -- `test_user_prompt_submit_mutates_last_user_message_in_array_input` — same, - for array-shaped `input`, asserting `sent_input[-1]["content"]`. -- `test_pre_tool_use_mutated_input_actually_reaches_tool_execute` — the tool's - `execute` callback records its received `params` into a list; the test - asserts the *tool implementation actually observed* the hook's - `mutated_input`. I traced the plumbing: `HookEntry` mutation config - (`hooks_types.py:117`, `mutations={"mutated_input": "tool_input"}`) maps the - hook's `mutated_input` return key onto `final_payload["tool_input"]`, which - `model_result.py:492-493` reads via `pre.final_payload.get("tool_input")` - and substitutes into `effective_call.arguments` before `execute_tool()` is - called. This is a genuine end-to-end proof, not an API-surface check. - -All four exercise actual behavior with concrete, falsifiable assertions -(request contents, request counts, executed-tool arguments) — not "call and -check nothing crashed." - -## Full battery re-run - -- `bash .upstreamer/scripts/verify.sh` → `PASS: 0 failures` (uv sync, ruff - check, ruff format, mypy, pytest tests/unit, all 31 required public-API - symbols present, version `0.8.0` matches upstream `package.json`, no - leaked TS artifacts, all repo-owned files present). -- `uv run pytest tests/unit tests/e2e -q` → `102 passed` (e2e tests skip - cleanly without `OPENROUTER_API_KEY`, per contract). - -## Broader sweep beyond the four claimed fixes - -Re-read the full upstream delta file list -(`git -C tmp/upstreamer/upstream diff --name-status adc7939..680bceb -- packages/agent/`) -and spot-checked every source file not already covered above: - -- **`conversation-state.ts` (versioned serialization, upstream #66)** — fully - present and correct in `conversation_state.py`: `CONVERSATION_STATE_VERSION`, - `InvalidStateError`, `UnsupportedStateVersionError`, - `serialize_conversation_state` / `deserialize_conversation_state` with the - same version-check-before-structural-validation ordering, same required - fields (`id`, `messages`, `status`), same "absence of `version` means v1" - policy. Covered by `tests/unit/test_conversation_state_serialization.py`. -- **`call-model.ts` / `async-params.ts`** — `strict_final_response` and - `hooks` (via `resolve_hooks`) are both threaded through `call_model.py` and - `async_params.py`'s reserved-key list, matching upstream's added fields. -- **`tool.ts` / `tool-types.ts` (the other ~85% of their diffs)** — almost - entirely TypeScript generic-variance engineering (`TContext` → `TCtx` / - `ContextFromSchema`, `bivarianceHack` method-syntax tricks so concretely - typed tools stay assignable to the wide `Tool` union). Zero runtime - behavior attached; correctly not chased in Python per Idiomatic Divergence - #4 ("Typing is looser... do not contort the runtime to chase a type-level - feature"). -- **`reusable-stream.ts`** — adds a `get isComplete()` getter. Not part of - the Required Public API list and not load-bearing (it's a cache-hit - optimization signal); not found ported 1:1 in `reusable_stream.py`, but - this is pre-existing from before this delta's scope and cosmetic (no - behavioral test depends on it upstream either). -- **`tool-orchestrator.ts` (`executeToolLoop`)** — upstream added a `source` - field here too, but this function is dead code in upstream itself: it is - not exported from `index.ts` and not imported by any other upstream - source file (only mentioned in a code *comment* in one test). The port's - `tool_orchestrator.py` has always been (since the original 0.7.2 port, not - this delta) a thin re-export shim (`ModelResult`, `execute_tool`, - `partition_tool_calls`) rather than a translation of `executeToolLoop`, so - there's no `source`-field gap to fix — the shim never had the field to - begin with, and the real tool-execution path (`model_result.py` + - `tool_executor.py`) does have `source` correctly threaded (see Finding 2). - Not a new gap; pre-existing and inconsequential since the dead code isn't - reachable either upstream or downstream. -- **Hooks system files** (`hooks-emit.ts`, `hooks-manager.ts`, - `hooks-matchers.ts`, `hooks-resolve.ts`, `hooks-schemas.ts`, - `hooks-types.ts`) — all six have Python counterparts - (`hooks_emit.py`, `hooks_manager.py`, `hooks_matchers.py`, - `hooks_resolve.py`, `hooks_schemas.py`, `hooks_types.py`). Spot-checked - `hooks_matchers.py` against `hooks-matchers.ts` line by line: `None`/wildcard, - exact string, compiled regex, and predicate-callable branches all match - (Python's `re.Pattern` has no `lastIndex`-style statefulness, so the - upstream comment about resetting `RegExp.lastIndex` has no Python - equivalent bug to guard against — correctly omitted, not a gap). - Session-id-per-emit threading (`hooks_manager.py:51-60`) is present and - tested (`test_hooks_manager.py::test_session_id_threads_per_emit_for_shared_manager`). -- **SessionEnd/drain on no-tools error paths** — behaviorally verified via a - throwaway script: a transport that raises on `send_async` for a *no-tools* - `call_model` still fires `SessionStart` then `SessionEnd` with - `reason: "error"` before the exception propagates (the `try/except/finally` - wrapping the entire `_run()` body guarantees this uniformly for tool and - no-tool paths alike). This is correct but has no dedicated committed test - pinning it — a pre-existing coverage gap, not a regression from this delta. -- **"Tool executes exactly once per round" regression** (upstream's - `tool-execution-once.test.ts`, guarding against a historical bug where - `handleApprovalCheck` pre-executed auto-approve tools and the main loop - re-executed them) — verified behaviorally via a throwaway script with a - mixed auto-tool + approval-gated-tool round and a `PermissionRequest` - `"allow"` decision: each tool's `execute` fired exactly once. Correct, but - there's no dedicated Python test file pinning this specific historical - regression by name — again a coverage gap, not a behavioral gap. -- **Approval/HITL resume ordering** (`function_call` before - `function_call_output`) — already covered by pre-existing - `test_parity_requirements.py::test_approval_pause_persists_tool_call_turn_and_resume_orders_output_after_call` - and `test_hitl_pause_persists_tool_call_turn_and_resume_orders_output_after_call`, - and by the new `test_manual_tool_pending_state.py::test_mixed_auto_and_manual_round_persists_auto_output_and_pauses_manual` - for the specific "auto output recorded before the pause" mixed-round case - this delta's contract calls out. No regression found. - -## Documentation gap (new finding, cosmetic) - -`upstreamer-changelog.md` was not updated for this 0.7.2 → 0.8.0 sync. It -still reads "Latest Sync ... against the current `@openrouter/agent` 0.7.2 -surface" and has no entry for the hooks system (upstream #7/#67, explicitly -called "the 0.8.0 headline" in the contract), MCP tool branding, versioned -conversation-state serialization (upstream #66), the `allow_final_response` -default-on `tool_choice: "none"` behavior change (upstream #68), or -`awaiting_client_tools` (upstream #64) — all of which *are* correctly -reflected in `README.md` (diff confirms new "Lifecycle Hooks" section, -updated Stop Conditions section, and a new manual-tools paragraph). The -contract's Output Shape section lists `upstreamer-changelog.md` alongside -`README.md` as this repo's own product surface / user-facing port notes; -leaving it stale after a headline feature release is a real (if purely -documentation-level) gap. Does not affect behavior, the public API, or any -Required Quality in eval.md — hence a warning, not a blocker. - -## Idiomatic Divergences section check - -The Stop-hook zero-cost-retry semantics are not a divergence — the port now -matches upstream's exact `runStopHook`/`shouldStopExecution` loop structure -(see Finding 1), so nothing needs to be added to -`.upstreamer/upstreamer.md`'s Idiomatic Divergences section for it. No other -undocumented divergence was found in this sweep; the existing six divergences -(Pydantic v2, async-first, generator tools, looser typing, Python-native -cancellation, structural protocols for `BeforeCreateRequestContext`) still -accurately describe the only permanent deltas from upstream. - -## Summary - -- Public API completeness: PASS (31/31 required symbols, plus the full - hooks/MCP-branding/versioned-state surface added in this delta). -- Version honesty: PASS (`0.8.0` == upstream `package.json` at target commit). -- Load-bearing loop: PASS, including the fixed Stop-hook zero-cost retry. -- Streaming / State / Approval-HITL ordering / Hooks / Compatibility helpers: - PASS, no regressions found beyond the four already-fixed items. -- Repo-owned files: PASS (LICENSE, README.md, pyproject.toml, scripts/upstream - all present and not clobbered). -- Warnings: `upstreamer-changelog.md` stale (documentation only); a few - historical-regression behaviors (SessionEnd-on-error, exactly-once tool - execution) are behaviorally correct but lack dedicated pinning tests; one - test docstring (`test_stop_hook_force_resume_then_falls_through_to_normal_tool_round`) - slightly mischaracterizes which code path runs, though its assertions are - correct; `reusable_stream.py` lacks upstream's new `is_complete` property - (non-load-bearing). - -**Verdict: PASS WITH WARNINGS.** All three prior FAIL findings are -substantively and verifiably fixed — not just superficially patched. The -remaining items are documentation or coverage nits with no behavioral impact. +FAIL + +# Port Parity Eval — Python (`@openrouter/agent` 0.8.0 → 1.2.0) — BANKRUPTCY + +Target upstream commit `50a1ec2a16f5c163136f81b208511b1c01e87874` (level with the +`@openrouter/agent@1.2.0` tag); last ported `680bceb4598f228d3e2ec58e2416e4335cdff059` +(0.8.0). Incremental run, force 0. + +**Outcome: bankruptcy declared. `.upstreamer/state.yaml` was NOT advanced.** +The mechanical verifier fails (version drift: `pyproject.toml=0.8.0` vs upstream +`1.2.0`) and two independent fresh-context evals returned `FAIL`. The next run +must retry this same delta. + +## Why the port cannot pass within this run + +The delta spans five upstream releases (0.9.0, 0.10.0, 0.11.0, 1.0.0/1.0.1, 1.1.0, +1.2.0; ~49k lines across `packages/agent`). Its 0.9.0 headline, upstream #90 +(`e8d7d6d`): unified `run()` tools with `sync`/`background`/`deferred` lifecycles, +`awaiting_async_tools`, `resume_tool_results`, `AsyncToolRegistry`, the universal +`task` tool and check-ins, steering (`send_to_task`, `queue_user_message`), +`tool.agent()` subagents, per-tool cancellation/timeouts (`signal`, +`tool_timeout_ms`), and tool concurrency (`tool_concurrency`, `Semaphore`). None of it +is ported. It is several thousand lines of engine changes plus about 10 upstream +test files. That is not a focused fix, and the contract does not list it as a +divergence. As a result, no version is both honest and verifier-passing: +- `0.8.0` fails the verifier. +- `1.2.0` overstates what was ported, which is a FAIL under eval.md's version-honesty rule. + +## Attempts + +1. **Initial port** of the delta, excluding #90 and MCP. The first fresh eval + (`FAIL`) found the following: + - #90 is missing. + - c610b6e's two-phase PreToolUse-before-approval gate is missing. This one is + security-relevant: a hook could mutate approved arguments without being + re-gated. + - `stream_replay` is not plumbed into `ModelResult`. + - There are smaller HITL/context/usage gaps. +2. **Focused fix:** ported the two-phase approval preparation. This adds the + `pre_tool_use_applied` marker, schema validation of mutated input, re-gating by + input-dependent approval sources, and no second PreToolUse on resume. The 5 + skipped upstream cases were un-skipped and pass. A second fresh eval re-ran + (`FAIL`) and confirmed the approval gate now matches upstream exactly. The + remaining blockers are #90 and version honesty. +3. A third attempt would not change the verdict. The remaining gap is a feature + port, not a fix, so the run stops here as the skill directs. + +## Second (final) fresh eval — verbatim + + +# Parity Eval (independent re-run): python-agent working tree vs `@openrouter/agent` @ 50a1ec2 + +- Last ported: `680bceb` (`@openrouter/agent` 0.8.0). Target: `50a1ec2` (`chore: version packages (#159)`). `git describe` says the target is exactly the `@openrouter/agent@1.2.0` tag. +- Delta: 50 upstream commits touching `packages/agent`, covering releases **0.9.0, 0.10.0, 0.11.0, 1.0.0, 1.0.1, 1.1.0, 1.2.0**. In `src` and tests: 142 files, about +49k/-1.3k lines. +- Port under review: the uncommitted working tree. 22 modules are modified, 3 are new (`doom_loop.py`, `tool_set.py`, `user_agent.py`), and 33 test files are new, plus `tests/vectors/doom-loop-fingerprints.json`. +- What I ran: + - `uv run pytest tests/unit -q`: 738 passed, 9 skipped, 23 xfailed. + - `.upstreamer/scripts/verify.sh`: **FAIL (1 failure)**. ruff, format, mypy, pytest, the 83% coverage floor and the 31-symbol API check all pass. The one failure is version drift. + +## Verdict rationale (short) + +The work that was ported is mostly faithful, and much of it is good: +- doom-loop detection +- the c610b6e approval split into two phases +- approval-decision validation (1601bff/1008d7e) +- stopWhen-before-approval (844845f) +- truncation guard, `tool_choice` relaxation, `next_turn_params.tool_choice` +- `get_usage`, ToolSet, user agent +- replay compaction in `ReusableStream` and the broadcaster + +But the tree is **not** a port of 1.2.0. It leaves out the headline of 0.9.0 (`e8d7d6d`: unified `run()` tools, background/deferred lifecycles, `resumeToolResults`, the universal `task` tool, steering, `tool.agent()` subagents, per-tool cancellation/timeouts, tool concurrency). That feature is not a documented divergence in the contract. + +The declared version also fails mechanically: it stays at 0.8.0 while upstream is 1.2.0. I can't find a version that is both honest and verifier-clean: +- 0.8.0 understates the content and fails the verifier. +- 1.2.0 would overstate it, and the eval says that is a FAIL. + +Separately, the port has several strict-xfail "PORT GAP"s in HITL, state and usage behaviour. Advancing `state.yaml` to `50a1ec2` would mark e8d7d6d as ported, and the next sync would skip it for good. That is exactly the failure this eval exists to stop. + +--- + +## 1. Version honesty: FAIL + +- `pyproject.toml:11` is `version = "0.8.0"`. Upstream `packages/agent/package.json` at the target is `"version": "1.2.0"`. verify.sh: `FAIL: version drift: pyproject.toml=0.8.0, upstream @openrouter/agent=1.2.0`. +- **0.8.0 is not an accurate label for the tree either.** The tree ships behaviour from 0.9.0 through 1.2.0: + - doom loop (0.9.0, `78c562e`) + - `get_usage` (`a629cf1`) + - `tool_choice` relaxation (`0efdbb0`) + - ToolSet (`8d2ed61`) + - `from_chat_messages` emitting `function_call` items (`dacf4d1`, 1.0.0) + - manual-tool wire schema (`4dce84e`, 1.0.0) + - user-agent identification (`ddab365`) + - truncation guard (`69aae63`) + - stopWhen-before-approval (1.0.1) + - MCP own JSON Schema (1.1.0) + - approval-decision hardening (1.2.0) +- **1.2.0 would overstate it.** The 0.9.0 minor (`e8d7d6d`, upstream CHANGELOG.md lines 324+) is almost entirely missing (see §2). The contract forbids setting the version "past what was actually ported", and the eval says overstating is a FAIL. +- No version both passes the verifier and tells the truth. So the tree can't be accepted at this target. This is a scope problem, not a one-line fix. +- Release-tag note (contract, Package Version): verify.sh says "ported tree is level with the 1.2.0 release tag (publishable)". That is true of the *target commit* and not of the *ported content*. Publishing this tree as 1.2.0 would ship a 1.2.0 that is missing 0.9.0's API. + +## 2. Public API completeness + +### Contract list: PASS +All 31 required symbols import from `openrouter_agent` (verify.sh check, and I confirmed by import). I spot-checked behaviour on the approval, state, hooks and stop paths below. + +### New upstream root exports in `packages/agent/src/index.ts` (680bceb..50a1ec2) + +**Present:** +- Doom loop: `DoomLoopMonitor`, `canonicalize_key_material`, `fingerprint_key_material`, `fingerprint_tool_call`, `detect_text_repetition`, `resolve_doom_loop_option`, `resolve_loop_key_material`, `DEFAULT_DOOM_LOOP_LADDER`, `DEFAULT_MAX_ESCALATIONS`, `MAX_CANONICALIZE_DEPTH` +- Async params: `TOOL_SET_SNAPSHOT` +- Tools and context: `mark_mcp`, `build_tool_execute_context`, `ToolContextStore`, `ToolEventBroadcaster` +- Tool sets: `create_tool_set`, `ToolSet` +- `ModelResult`: `get_usage`, `get_items_stream`, `get_doom_loop_verdict` + +**Missing at runtime (none appear in Idiomatic Divergences):** + +| Upstream export | Module | Python name per naming convention | +|---|---|---| +| `resumeToolResults`, `ToolTaskAlreadySettledError` | `inner-loop/resume-tool-results.ts` | `resume_tool_results`, `ToolTaskAlreadySettledError` | +| `AgentTranscriptSource` | `lib/agent-tool.ts` | `AgentTranscriptSource` | +| `AsyncToolRegistry` | `lib/async-tool-registry.ts` | `AsyncToolRegistry` | +| `TASK_TOOL_NAME`, `TaskToolInputSchema` | `lib/tool-check.ts` | same | +| `acquireAll`, `Semaphore` | `lib/tool-concurrency.ts` | `acquire_all`, `Semaphore` | +| `DEFAULT_TASK_LOG_LIMITS`, `ToolTask` | `lib/tool-task.ts` | same | +| `isAgentTool`, `isDeferredHandle`, `isLongRunningTool`, `isToolAsyncStartedEvent`, `isToolAsyncSettledEvent`, `isUnifiedTool` | `lib/tool-types.ts` | `is_agent_tool`, … | +| `tool.background`, `tool.deferred`, `tool.agent`; `tool({ run, lifecycle })` | `lib/tool.ts` | `tool()` has no `run`/`lifecycle` (`src/openrouter_agent/tool.py`) | +| `ModelResult.getAsyncTasks/cancelTask/sendToTask/queueUserMessage` | `lib/model-result.ts:7527-7573` | absent from `model_result.py` | +| `ConversationStatus 'awaiting_async_tools'`, `ConversationState.pendingAsyncTools` | `lib/tool-types.ts:1823,1922` | absent from `tool_types.ConversationState`. A TS blob carrying `pendingAsyncTools` is silently dropped by `deserialize_conversation_state`. | +| callModel options `signal`, `toolTimeoutMs`, `toolConcurrency`, `asyncTools`; per-tool `timeoutMs`, `maxConcurrency` | | absent | + +Upstream subpath exports added to `package.json` with no Python module: +- `./tool-concurrency`, `./async-tool-registry`, `./resume-tool-results`, `./tool-task`, `./tool-check`, `./agent-tool` +- `./mcp`, `./mcp/*` (see §9) + +The contract requires "one Python module per upstream lib module". `lib/agent-tool.ts`, `async-tool-registry.ts`, `async-tools.ts`, `tool-check.ts`, `tool-concurrency.ts`, `tool-task.ts` and `inner-loop/resume-tool-results.ts` have no counterpart. + +`ModelResult.queueUserMessage` is general steering, not specific to async tools, and it is also missing. + +**New naming mappings to add to the contract table** (derived by the port, all consistent with convention): +- `getUsage` → `get_usage` +- `getItemsStream` → `get_items_stream` +- `getDoomLoopVerdict` → `get_doom_loop_verdict` +- `createToolSet` → `create_tool_set` +- `markMcp` → `mark_mcp` +- `doomLoop` → `doom_loop` +- `loopKey` → `loop_key` +- `strictFinalResponse` → `strict_final_response` +- `preToolUseApplied` → `pre_tool_use_applied` +- `TOOL_SET_SNAPSHOT` unchanged + +## 3. Load-bearing loop: parity on what exists, with one pre-existing undocumented divergence + +- I read `ModelResult._run` (`model_result.py:1878-2206`): + - Request sequence, accumulated input via `_build_followup_request`, stop-condition evaluation, the `allow_final_response` final turn with stubs for unexecuted calls, and the empty-final retry all match upstream. + - `max_output_tokens` truncation skips execution (`stream_transformers.py`, `test_max_output_tokens_truncation.py`). + - `next_turn_params.tool_choice` becomes the new caller-level policy (`_apply_next_turn_params`, :2245). + - Forced `tool_choice` is relaxed on follow-ups. +- **Divergence (pre-existing, undocumented, now asserted upstream):** + - Upstream executes one round's calls concurrently (`Promise.allSettled`, `model-result.ts:3093` and :3973). It was concurrent before this delta too. + - The port runs them one by one: `_execute_round` (`model_result.py:1812`, a `for call in calls: await …`) and `_execute_auto_approve_tools` (:1470). + - Upstream `tool-cancellation.test.ts` "unbounded by default (all calls run at once)" pins concurrency. It is unported. + - Output order matches; wall-clock and overlapping side-effect behaviour does not. + - This should be fixed, or added as a documented divergence. +- No cancellation of individual tools or of the run by signal, and no tool timeouts (`tool_timeout` error output). `cancel()` (`model_result.py:2550`) cancels the asyncio task. That is acceptable under divergence 5 for whole-run cancel. Per-tool `timeout_ms` / `tool_timeout_ms` and the `tool_timeout` error code have no equivalent. + +## 4. Streaming: PASS on the ported surface + +- `ReusableStream` and `ToolEventBroadcaster` replay compaction and backlog release (`17418f7`, `250e64e`) are ported, with `test_reusable_stream.py` (12 tests) and `test_replay_buffer_compaction.py` (8 tests). +- `turn.start` / `turn.end` stay paired per turn (:1961-1992). +- The late-close terminal-event case for PostModelCall (`test_hooks_post_model_call.py`) asserts exact counts and payload. +- **Gap:** upstream `model-result-replay-compaction.test.ts` (6 cases: terminal response after the active consumer trims history, `streamReplay` stripped from follow-up and resolved requests, cached terminal response, cleanup-failure handling) has no Python counterpart. These streaming invariants are FAIL-class under the eval. Port them, or show they are covered elsewhere. +- No `tool.async_started` / `tool.async_settled` events (they come with the missing async tools). + +## 5. State: parity on the ported surface, one gap + +- Versioned serialization works: + - unchanged `CONVERSATION_STATE_VERSION` + - an unsupported version raises before shape checks + - `pre_tool_use_applied` is written only when `True`, and legacy blobs load as `False` (`conversation_state.py:104-109`, `:169-180`) + - tests: `test_conversation_state_serialization.py:148-187` +- `tool_requires_approval` (`conversation_state.py:227-268`) matches c610b6e: + - the call-level check gets the normalized copy when parsing succeeds and the raw call otherwise + - the tool-level predicate gets parsed args + - a schema-invalid or non-mapping parse fails closed +- **Gap:** `ConversationState` has no `pending_async_tools` and no `awaiting_async_tools` status (see §2). +- Minor: `deserialize_conversation_state` builds `ParsedToolCall(**item)`, so an unknown key in a pending call raises `TypeError` instead of `InvalidStateError`. +- **Gap (strict xfail, state/approval):** `test_get_usage.py:316` `test_does_not_advance_an_approval_resumed_run_when_read_before_the_loop`. + - Upstream `getUsage()` (`model-result.ts:7704-7712`) only observes an approval-resumed run (`isResumingFromApproval` guard). + - The port's `get_usage()` (`model_result.py:2354`) awaits `_ensure_task()`, which drives the entire run: it executes approved tools, makes further model requests and marks the conversation complete. + - So reading usage has side effects, including spend, that a TS user doesn't get. + - This is a behavioural gap in new delta API on the approval/state path. It is pinned by an xfail but is not a contract divergence. + +## 6. Approval / HITL ordering: c610b6e is faithfully ported; HITL output gaps remain + +- **Two-phase gate.** + - `_handle_approval_check` / `_resolve_approval_phases` / `_prepare_after_initial_approval` (`model_result.py:1493-1618`, :808-839) match upstream `handleApprovalCheck` / `resolveApprovalPhases` / `prepareAfterInitialApproval` (`model-result.ts:3272-3384`, :3188-3258, :2871-2904) step for step. + - Phase 1 runs partition plus PermissionRequest on the model-issued args. + - Phase 2 runs PreToolUse for auto-resolvable survivors. A mutated input is schema-validated, then re-gated once (mutated-gate key) by input-dependent sources only (`_mutated_input_requires_approval`). + - A call that pauses on the second gate is persisted with `pre_tool_use_applied=True` (:1557-1560, matching upstream :3367-3372). + - On resume: + - an approved call without the marker runs PreToolUse and may re-pause (:1697-1709) + - one with the marker skips PreToolUse (`run_pre_tool_use=False`) + - a rejected call with the marker emits `PostToolUseFailure` (:1731-1734), matching upstream :6079-6163 + - **This means PreToolUse runs before the final approval decision for survivors, and runs only after the human decision for calls pending from phase 1.** That is upstream's ordering exactly. The marker is honoured. + - Doom-loop classification runs before partitioning (`_classify_initial_approval_calls`, matching `classifyInitialApprovalCalls`). +- **Mixed turns.** + - Auto-executable siblings run before the pause and are stored as `unsent_tool_results` along with hook denials. + - With `allow_final_response is False` and a met `stop_when`, they are recorded as "skipped" on turn 0 only (844845f; same call sites as upstream :6417-6421). + - Resume appends unsent outputs after the persisted messages (:1778-1781). + - The approval gate is now also enforced on the `allow_final_response` path (:2083). +- **Approval decisions** (1601bff/1008d7e): + - the same ID in both lists raises (:240-248) + - a resume with no decision stays paused + - HITL output resumes still work + - tests: `test_approval_gate_regressions.py` (51 cases vs 42 upstream; 2 skipped) +- **Gate identity:** + - The port keys gates on `(id(response), index)`. Upstream uses a per-response occurrence counter (`approvalResponseKey` / `assignApprovalCallOccurrences`, :2752-2789). + - This is safe today because every response stays alive in `self._steps`, so `id()` can't be reused within a run. + - The subset-call contract is skipped with a reason (`test_approval_gate_regressions.py:975`). + - It is fragile and undocumented. Warning only. +- **HITL gaps (strict xfails, pre-existing but now exposed; FAIL-class area):** in `test_hitl_tool.py`: + - `:382` — `apply_on_response_received_hooks` (`tool_executor.py`) validates and rewrites caller outputs for *any* client tool. Upstream only touches HITL tools (`buildHitlToolMap`, `tool-executor.ts:1018`). A regular tool's caller-supplied output that doesn't match its schema gets turned into an error wrapper. + - `:564` — when a hook throws on a content-array output, the port emits a JSON string. Upstream emits a content array (`formatHookError`). + - `:861` — the port serializes the schema-validated hook return and drops extra keys. Upstream serializes the raw return. +- `test_tool_context.py:152,178` (strict xfails): `set_context` does not validate against the context schema, and is not a no-op when the schema is absent. Upstream `tool-context.ts:113-133, 237-240`. + +## 7. Hooks: PASS + +- Session id is threaded per emit (`session_id=self._session_id`). +- `SessionEnd` and drain sit in `finally` (:2197-2206), with teardown errors downgraded to warnings. +- A `DoomLoopDetected` hook and `"doom_loop"` SessionEnd reason are present. +- The 231fb65 tool call is threaded into the hook context, and `hooks` / `active_tools` are stripped from the outgoing request on both resolution paths (`test_hooks_session_lifecycle.py`; asserts exact event pairs). +- `test_hooks_session_lifecycle.py` and `test_hooks_post_model_call.py` port only the delta cases. That is acceptable because the base cases live in `test_model_result_hooks.py`, but it breaks the 1:1 file mapping. + +## 8. Doom-loop integration: PASS (well covered) + +- `doom_loop.py` (1289 lines) is wired into `_classify_initial_approval_calls`, `_begin_doom_loop_round`, `_check_doom_loop_for_response`, the steer flush (:2166), escalation dispatch overrides (:1981), the stop seal (:2049, :2099, :2151), and state persistence/restore (:503-544). Steer is deferred across HITL pauses, as upstream does. +- Upstream fingerprint vectors are shared (`tests/vectors/doom-loop-fingerprints.json`). +- Test counts match or exceed upstream: + +| Test file | Port | Upstream | +|---|---|---| +| doom-loop | 96 | 66 | +| integration | 21 | 21 | +| fanout | 27 | 27 | +| remediation | 23 | 21 | +| escalation | 16 | 16 | +| public-api | 6 | 6 | + +- The integration fixtures build on `tests/_fixtures.py` builders (`Script` wraps `make_response`/`function_call_item`) and assert execution counts. +- Upstream's "late-result delivery as forward progress" can't apply here, because there are no async tools. + +## 9. Contract scope vs upstream moving MCP into `packages/agent/src/mcp` + +- Since `787cbf8` (#102), the MCP client lives inside `packages/agent`: + - `src/mcp/*`, 22 files + - subpath exports `@openrouter/agent/mcp`, `/mcp/create-mcp-tools`, `/mcp/types`, `/mcp/schema`, `/mcp/cache` + - `@openrouter/mcp` is now a compatibility facade +- Read literally, the contract's Scope ("Port only `packages/agent/`") now *includes* MCP. Its intent ("`packages/mcp/` … Not ported … Adding it is a deliberate contract change") excludes it. +- The port took the intent route: no `mcp` subpackage. It kept only root-level MCP touchpoints (`mark_mcp`, `is_mcp_tool`, `source: "mcp"`, and the MCP tool's own JSON Schema on the wire from 907beec, in `test_mcp_tool_branding.py`). +- That is a reasonable reading, but it is **undocumented**. The contract needs an explicit amendment that names `packages/agent/src/mcp/**` and `tests/unit/mcp/**` as out of scope. Without one, every future sync is ambiguous. +- verify.sh's parity diff only globs `tests/unit/*.test.ts`, so the 16 MCP test files are invisible to it: + - `build-tools`, `cache`, `call-tool-shape`, `create-mcp-tools`, `elicitation`, `json-schema-to-zod`, `list-tools-pagination`, `loop-key`, `mcp-connection`, `mcp-sdk`, `protocol-era`, `rehydrate`, `resource-tools`, `result-mapper`, `version`, `wire-input-schema` +- Severity: **warning, if the contract is amended**. Otherwise it is an unported in-scope module. +- `mcp/loop-key.test.ts` and `mcp/wire-input-schema.test.ts` also test root-level behaviour: `loopKey` on wrapped tools, and the wire schema. Check that their root-level assertions are covered by `test_doom_loop*.py` and `test_manual_tool_wire_schema.py`. + +## 10. Test parity + +### Unported upstream test files (`comm -23`, 33 files plus 16 under `tests/unit/mcp/`) + +**New in this delta and FAIL-class** (they cover the tool loop, streaming, approval or state): + +| Upstream file | Cases | Invariant | Severity | +|---|---|---|---| +| `async-tool-background.test.ts` | 8 | background lifecycle, pending placeholder, `tool_task_result` injection, `onRunEnd` | FAIL (tool loop) | +| `async-tool-deferred.test.ts` | 19 | deferred pause `awaiting_async_tools`, `.resolve/.fail/.cancel`, double-settle error, tool_choice on resume | FAIL (state / HITL-like pause) | +| `async-tool-registry.test.ts` | 5 | registry tracking/settling | FAIL (state) | +| `task-tool-actions.test.ts` | 11 | universal `task` tool check/steer/result/cancel | FAIL (tool loop) | +| `task-tool-integration.test.ts` | 7 | task tool inside the loop | FAIL (tool loop) | +| `tool-check.test.ts` | 13 | custom `check` config dispatch | FAIL (tool loop) | +| `agent-tool.test.ts` | 9 | `tool.agent()` subagents | FAIL (tool loop) | +| `steering.test.ts` | 4 | `ToolTask` inbox, `sendToTask` | FAIL (tool loop) | +| `tool-cancellation.test.ts` | 10 | `ctx.signal`, `tool_timeout`, run abort, `cancel()` aborts tools, **concurrency incl. unbounded-by-default**, semaphore release | FAIL (tool loop) | +| `run-cancellation.test.ts` | 7 | run `signal` into every dispatch, pre-aborted fails before network, `timeoutMs` composition | FAIL (loop / streaming). Partly divergence 5, but the invariants (no dispatch after abort, per-dispatch timeout) still need Python equivalents. | +| `model-result-replay-compaction.test.ts` | 6 | terminal response after replay trim, `streamReplay` stripping | FAIL (streaming) | +| `startup-import-closure.test.ts` | 1 | optional deps not in the root import closure | warning | +| `package-boundaries.test.ts` | 0 | packaging only | cosmetic | + +**Pre-existing (unported at 680bceb as well, unchanged in this delta).** These were accepted by the previous eval, presumably as covered by `test_model_result_hooks.py` and others. I note them, and did not re-grade them as delta gaps: +- `assistant-message-images`, `build-tool-call-stream`, `conversation-state` (42 cases), `create-tool`, `get-items-stream-manual-tools` +- `hooks-contract-fixes`, `hooks-emit(-adversarial)`, `hooks-manager-adversarial`, `hooks-matchers-adversarial`, `hooks-resolve(-adversarial)`, `hooks-types-adversarial`, `hooks-void-schema` +- `manual-tool-calls-adversarial`, `model-result-stream-detection`, `schema-sanitization`, `shared-context`, `tool-type-compat`, `user-input-persistence` + +The 1:1 mapping is still unmet for them. `conversation-state.test.ts` (42 cases) and `user-input-persistence.test.ts` cover state and are worth closing. + +Type-level `.test-d.ts` files (`async-tool-types`, `filter-tools-by-ids`, `full-field-examples`, `manual-tool-wire-schema`, `message-compat-types`, `resolved-tools`, `server-tool-id`, `tool-name-correlation`, `tool-shared-name`) are a warning only (divergence 4). + +### Quality of new tests + +- Generally strong: + - order and count assertions on event sequences + - exact execution counts (`ExecutionCounter`, `Counter`) + - requests asserted by index + - fixtures built from `tests/_fixtures.py`; bespoke transports (`FailingClient(QueuedClient)`, async-generator streams) still use the shared builders +- Flags: + - `test_tool_name_events.py:212`: `assert len(tool_events) >= 1`. The invariant is which events fire and how many. + - `test_tool_name_events.py:1-12` docstring says `toolName` is "pinned by a strict xfail", but there is no xfail in that file now. The docstring is stale. + - `test_tool_orchestrator.py`: 17 of 27 cases are strict xfails against a non-existent `execute_tool_loop` / result-helper API. `lib/tool-orchestrator.ts` is not a root export and is unreferenced upstream, so this is a warning. But the file is 63% xfail, and it is xfail scaffolding, not coverage. + - Skips with reasons naming unported async/agent/cancellation features: `test_server_tool.py:114,119`, `test_tool_context.py:118,123`, `test_call_model_active_tools.py:74`, `test_approval_gate_regressions.py:717`, `test_tool_choice_relaxation.py:320` (the `interrupted` status path). These confirm the gaps in §2 and §3. + +### Coverage floor +`--cov-fail-under=83` in `.github/workflows/ci.yaml:80` is unchanged and matches verify.sh. It was not lowered. + +## 11. Divergences and repo-owned files + +- **Undocumented divergences (findings):** + - async/unified tools, task/steering/agent tools, tool cancellation/timeout/concurrency (§2) + - sequential round execution (§3) + - `get_usage` drives the run (§5) + - HITL `on_response_received` differences (§6) + - `set_context` schema handling (§6) + - MCP subpath (§9) + - Skip/xfail reasons call several of these "Out of scope", but the contract has no such scope entry. +- **Repo-owned files intact:** `.github/`, `LICENSE`, and `pyproject.toml` (`[project.urls]`, classifiers, distribution name `openrouter-agent-sdk`) are unchanged. The user-agent token uses the PyPI name (divergence 7, `user_agent.py`). +- **Docs:** + - README was updated for doom loop and the PreToolUse ordering, and its claims match the code. + - `upstreamer-changelog.md` was **not** updated for this delta, so there is no user-facing note that 0.9.0's async-tool feature set is absent. + +--- + +## What's needed to pass + +1. **Close the scope gap, or narrow the target.** Choose one: + - (a) Port `e8d7d6d` and its follow-ups: + - `tool(run=…, lifecycle=…)`, `tool.background`, `tool.deferred`, `tool.agent` + - `AsyncToolRegistry`, `resume_tool_results`, `ToolTaskAlreadySettledError` + - the universal `task` tool (`TASK_TOOL_NAME`, `TaskToolInputSchema`, `check` config), `ToolTask` with steering inbox and `DEFAULT_TASK_LOG_LIMITS` + - `Semaphore`/`acquire_all`, `tool_concurrency`, `max_concurrency` + - per-tool `timeout_ms` / `tool_timeout_ms`; `ModelResult.get_async_tasks/cancel_task/send_to_task/queue_user_message` + - `pending_async_tools` and the `awaiting_async_tools` / `interrupted` statuses + - the `is_*` guards and `tool.async_*` events + - Then set `version = "1.2.0"`. + - (b) Retarget the run at the last commit before `e8d7d6d`, so the version and content can be made consistent. Then port forward in a later sync. + - (c) Amend the contract with an explicit, reasoned Idiomatic Divergence or Scope exclusion for the async-tool system. Only then can the version be set to 1.2.0 without overstating. +2. **Make verify.sh pass.** The version must equal upstream `package.json` (1.2.0) *and* be honest, which follows from item 1. +3. **Fix `get_usage()`** so it only observes on an approval-resumed run (upstream `model-result.ts:7704-7712`). Turn `test_get_usage.py:316` from xfail into a passing test. +4. **Fix the HITL `apply_on_response_received_hooks` gaps** (`test_hitl_tool.py:382, :564, :861`) and `set_context` schema handling (`test_tool_context.py:152, :178`). Otherwise, record each one as a documented divergence with its reasoning. +5. **Port or justify sequential round execution.** Either run round calls concurrently (matching `Promise.allSettled`) or document it as a divergence. Port the concurrency cases of `tool-cancellation.test.ts`. +6. **Port `model-result-replay-compaction.test.ts`** (streaming), and add Python equivalents of the `run-cancellation.test.ts` invariants under divergence 5. +7. **Amend the contract's Scope** to say explicitly whether `packages/agent/src/mcp/**` and `tests/unit/mcp/**` are in or out. Extend verify.sh's parity diff to recurse into `tests/unit/**`. +8. **Clean up the tests.** Replace `len(tool_events) >= 1` in `test_tool_name_events.py:212` with an exact count, and fix the stale docstring. +9. **Update `upstreamer-changelog.md`** with the delta and every parity gap left open. Add the new naming mappings listed in §2 to the contract table. + +## Recommended human action + +1. **Decide on #90 (`e8d7d6d`) explicitly. Pick one:** + - (a) Schedule a dedicated port of the async-tool system and its follow-ups. + Candidate modules: `tool_task`, `tool_check`, `async_tool_registry`, + `async_tools`, `agent_tool`, `tool_concurrency`, `resume_tool_results`, plus + `ModelResult` additions. Tests: async-tool-background/deferred/registry, + task-tool-actions/integration, tool-check, agent-tool, steering, + tool-cancellation, run-cancellation. Then re-run this sync. + - (b) Amend the contract to make the async-tool system a documented, versioned + divergence, and define a version strategy that does not claim ≥0.9.0 without it. +2. **Amend the contract scope for MCP.** Upstream moved it into + `packages/agent/src/mcp` (`@openrouter/agent/mcp`), so "port only + `packages/agent/`" now literally includes it. This run treated it as out of + scope, following the contract's intent. Also make `verify.sh`'s parity check + recurse into `tests/unit/**`. +3. **Add the new naming mappings to the contract table** (listed in the final + report). +4. **Keep the partial work in this run's working tree.** It is verified green + apart from the version check. Choose one: + - land it on a branch the next sync builds on; + - discard it, in which case the next run re-ports it. + + Do not publish to PyPI from it. +5. **Known remaining non-#90 gaps** (strict-xfail pinned in tests): + - `get_usage()` drives an approval-resumed run instead of only observing it. + - Three HITL `on_response_received` output-handling differences. + - `set_context` schema validation and the no-schema no-op. + - `tool_orchestrator.execute_tool_loop` and its four result helpers. + - `stream_replay` is accepted but not wired into `ModelResult`'s event buffer. + - Round tool calls run sequentially; upstream runs them concurrently. diff --git a/README.md b/README.md index e0372b2..ecbce0e 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ hooks.on(HookName.SessionEnd.value, HookEntry(handler=lambda payload, ctx: print result = call_model(client, {"model": model, "input": prompt, "tools": tools, "hooks": hooks}) ``` -`SessionStart` fires once per run with a config summary; `SessionEnd` fires once with aggregated `total_usage` (summed across every `PostModelCall`) and is guaranteed to fire — and any pending async hook work drained — even when a no-tools stream raises. `PreToolUse` can block a call (`{"block": "reason"}`) or mutate its input (`{"mutated_input": {...}}`); `PermissionRequest` can pre-empt the approval gate with `{"decision": "allow" | "deny" | "ask_user"}`; `Stop` can force the loop to keep going past a `stop_when` hit with `{"force_resume": True, "append_prompt": "..."}`. A `HooksManager` instance is safe to share across concurrent `call_model` runs — session identity is threaded per emit, not stored as manager-level mutable state. +`SessionStart` fires once per run with a config summary; `SessionEnd` fires once with aggregated `total_usage` (summed across every `PostModelCall`) and is guaranteed to fire — and any pending async hook work drained — even when a no-tools stream raises. `PreToolUse` can block a call (`{"block": "reason"}`) or mutate its input (`{"mutated_input": {...}}`). It runs once per call, after the initial approval check and before execution, so a mutated input is validated against the tool's `input_schema` and re-gated by any input-dependent approval source (a callable `require_approval`, tool- or call-level); a call that pauses on that second gate is persisted with its effective arguments and `pre_tool_use_applied=True`, and the resume does not run `PreToolUse` again. `PermissionRequest` can pre-empt the approval gate with `{"decision": "allow" | "deny" | "ask_user"}`; `Stop` can force the loop to keep going past a `stop_when` hit with `{"force_resume": True, "append_prompt": "..."}`. A `HooksManager` instance is safe to share across concurrent `call_model` runs — session identity is threaded per emit, not stored as manager-level mutable state. ## Stop Conditions @@ -191,6 +191,31 @@ The built-ins mirror the TypeScript package and OR together when provided as a l When a stop condition fires while the model is still emitting tool calls, `call_model` makes one more turn with `tool_choice="none"` by default (tools stay in the request so the prompt-cache prefix survives) so the run ends with a natural-language answer. `allow_final_response` tunes this: `True` or omitted appends `DEFAULT_FINAL_RESPONSE_DIRECTIVE` as a user message, a non-empty string replaces the wording, `""` appends nothing, and `False` disables the extra turn entirely. +## Doom-Loop Detection + +Opt in with `doom_loop=True` (defaults) or a config dict to catch a run that keeps spending without progress: the same tool call re-issued with identical arguments (including empty and invalid-JSON calls), repeated server-tool calls, or repeated text. Detection is deterministic and feeds a ladder — `observe` (emit `DoomLoopDetected`), `steer` (inject guidance as a user message before the next request), `escalate` (one-turn `model` swap and/or forced `openrouter:advisor` consult, capped by `max_escalations`), `block` (refuse the call with an error output, before `PreToolUse`), `stop` (halt before any further model request; `SessionEnd` reason `"doom_loop"`). Defaults: observe at 2, block at 3, stop at 6. + +```python +result = call_model(client, { + "model": "openai/gpt-4o-mini", + "input": "Find it.", + "tools": [tool(name="web_search", input_schema=Query, execute=search, + loop_key=lambda args: args["query"].strip().lower())], + "doom_loop": {"ladder": {"steer": 2, "block": 3, "stop": 5}}, +}) +verdict = await result.get_doom_loop_verdict() # the stop verdict, or None +``` + +`loop_key` declares a tool's call identity (`False` exempts the tool; a callable returning `None` exempts that call). A `DoomLoopDetected` handler can return `{"override_action": ...}`. Detector state persists in `ConversationState.doom_loop`, so a resumed doom loop is still a doom loop. + +## Usage, Tool Sets, and Tool Choice + +- `await result.get_usage()` returns aggregate token/cost totals across every model call the run made (same shape as `SessionEnd`'s `total_usage`); it never raises. +- `active_tools=[...]` on `call_model` filters which client tools are sent to (and executable by) the model; server tools always pass. `create_tool_set(tools)` builds declarative `activate` / `deactivate` / `activate_when` / `deactivate_when` rules, and `tool_set.resolve().call_model` is a `{"tools", "active_tools"}` mapping to spread into the request. +- A forced `tool_choice` (`"required"`, a specific tool, or `allowed_tools` with `mode: "required"`) relaxes to `auto` after it produces a tool call, so the model can answer in text. Tools can change the next turn's `tool_choice` via `next_turn_params={"tool_choice": fn}`. +- `tool(..., strict=True)` passes `strict` through to the wire definition; `tool(..., wire_input_schema={...})` sends a caller-owned JSON Schema instead of the generated one; `server_tool(config, {"id": ...})` overrides the default `server:` id. +- A response truncated at `max_output_tokens` never executes its (cut-off) tool calls. + ## Format Compatibility Use `from_claude_messages` / `to_claude_message` for Anthropic-style messages and `from_chat_messages` / `to_chat_message` for OpenAI Chat-style messages. Content that cannot be represented directly is carried as `unsupported_content` instead of being silently discarded. diff --git a/src/openrouter_agent/__init__.py b/src/openrouter_agent/__init__.py index 5036ce4..eae8bb6 100644 --- a/src/openrouter_agent/__init__.py +++ b/src/openrouter_agent/__init__.py @@ -80,6 +80,8 @@ ) from .hooks_manager import HooksManager from .hooks_schemas import ( + DoomLoopDetectedPayload, + DoomLoopDetectedResult, HookDefinition, HookName, HookRegistry, @@ -222,6 +224,74 @@ tool_has_approval_configured, ) from .turn_context import build_turn_context, normalize_input_to_array +from .doom_loop import ( + DEFAULT_DOOM_LOOP_LADDER, + DEFAULT_MAX_ESCALATIONS, + MAX_CANONICALIZE_DEPTH, + DoomLoopAction, + DoomLoopCallRecord, + DoomLoopConfig, + DoomLoopDetectorKind, + DoomLoopEscalationConfig, + DoomLoopLadder, + DoomLoopMonitor, + DoomLoopOption, + DoomLoopRoundCall, + DoomLoopSerializedState, + DoomLoopStreak, + DoomLoopTextOptions, + DoomLoopVerdict, + LoopKeyResolution, + ResolvedDoomLoopConfig, + ResolvedEscalationConfig, + TextRepetitionResult, + canonicalize_key_material, + detect_text_repetition, + fingerprint_key_material, + fingerprint_tool_call, + resolve_doom_loop_option, + resolve_ladder_action, + resolve_loop_key_material, +) +from .tool_set import ( + TOOL_SET_SNAPSHOT, + ActivationInput, + ActivationPredicate, + ConditionalMode, + Directive, + ResolvedToolSnapshot, + SituationConditionalRule, + SituationConditionalRuleObject, + SituationConfig, + StatusReason, + ToolSet, + ToolStatusEntry, + create_tool_set, + get_server_tool_id, + get_tool_id, + strip_tool_set_snapshot_metadata, +) +from .user_agent import ( + AGENT_PACKAGE_NAME, + AGENT_USER_AGENT, + PACKAGE_VERSION, + SDK_USER_AGENT, + agent_user_agent_token, + append_agent_user_agent, + build_agent_user_agent, + get_package_version, +) +from .reusable_stream import ( + BUFFER_COMPACTION_MIN_HEAD, + ReplayConsumer, + ReusableReadableStream, + StreamReplay, +) +from .turn_context import is_input_item, output_to_input_items +from .item_types import NewAssistantMessageItem, NewDeveloperMessageItem, NewSystemMessageItem +from .chat_compat import ChatMessageInputItem +from .anthropic_compat import ClaudeMessageInputItem +from .stream_transformers import is_truncated_at_max_output_tokens, response_has_tool_calls RequestOptions: TypeAlias = Mapping[str, Any] @@ -462,4 +532,70 @@ def before_create_request(self, hook_ctx: BeforeCreateRequestContext, request: A "tool_has_approval_configured", "tool_requires_approval", "update_state", + "DEFAULT_DOOM_LOOP_LADDER", + "DEFAULT_MAX_ESCALATIONS", + "MAX_CANONICALIZE_DEPTH", + "DoomLoopAction", + "DoomLoopCallRecord", + "DoomLoopConfig", + "DoomLoopDetectedPayload", + "DoomLoopDetectedResult", + "DoomLoopDetectorKind", + "DoomLoopEscalationConfig", + "DoomLoopLadder", + "DoomLoopMonitor", + "DoomLoopOption", + "DoomLoopRoundCall", + "DoomLoopSerializedState", + "DoomLoopStreak", + "DoomLoopTextOptions", + "DoomLoopVerdict", + "LoopKeyResolution", + "ResolvedDoomLoopConfig", + "ResolvedEscalationConfig", + "TextRepetitionResult", + "canonicalize_key_material", + "detect_text_repetition", + "fingerprint_key_material", + "fingerprint_tool_call", + "resolve_doom_loop_option", + "resolve_ladder_action", + "resolve_loop_key_material", + "TOOL_SET_SNAPSHOT", + "ActivationInput", + "ActivationPredicate", + "ConditionalMode", + "Directive", + "ResolvedToolSnapshot", + "SituationConditionalRule", + "SituationConditionalRuleObject", + "SituationConfig", + "StatusReason", + "ToolSet", + "ToolStatusEntry", + "create_tool_set", + "get_server_tool_id", + "get_tool_id", + "strip_tool_set_snapshot_metadata", + "AGENT_PACKAGE_NAME", + "AGENT_USER_AGENT", + "PACKAGE_VERSION", + "SDK_USER_AGENT", + "agent_user_agent_token", + "append_agent_user_agent", + "build_agent_user_agent", + "get_package_version", + "BUFFER_COMPACTION_MIN_HEAD", + "ReplayConsumer", + "ReusableReadableStream", + "StreamReplay", + "is_input_item", + "output_to_input_items", + "NewAssistantMessageItem", + "NewDeveloperMessageItem", + "NewSystemMessageItem", + "ChatMessageInputItem", + "ClaudeMessageInputItem", + "is_truncated_at_max_output_tokens", + "response_has_tool_calls", ] diff --git a/src/openrouter_agent/anthropic_compat.py b/src/openrouter_agent/anthropic_compat.py index 49921ee..313816e 100644 --- a/src/openrouter_agent/anthropic_compat.py +++ b/src/openrouter_agent/anthropic_compat.py @@ -4,13 +4,18 @@ from ._utils import get_field, json_dumps, json_loads_maybe +ClaudeMessageInputItem = Dict[str, Any] +"""An OpenResponses input item emitted by `from_claude_messages`: a user or +assistant message item, ``function_call``, ``function_call_output``, or +``image_generation_call`` item.""" + def _unsupported(original_type: str, data: Any, reason: str) -> Dict[str, Any]: return {"original_type": original_type, "data": data, "reason": reason} -def from_claude_messages(messages: Sequence[Mapping[str, Any]]) -> List[Dict[str, Any]]: - output: List[Dict[str, Any]] = [] +def from_claude_messages(messages: Sequence[Mapping[str, Any]]) -> List[ClaudeMessageInputItem]: + output: List[ClaudeMessageInputItem] = [] for message in messages: role = str(message.get("role", "user")) content = message.get("content", "") diff --git a/src/openrouter_agent/async_params.py b/src/openrouter_agent/async_params.py index 49141b8..b857b57 100644 --- a/src/openrouter_agent/async_params.py +++ b/src/openrouter_agent/async_params.py @@ -11,6 +11,8 @@ class CallModelInput(TypedDict, total=False): model: str input: Any tools: Sequence[Any] + active_tools: Sequence[str] + stream_replay: str stop_when: Any state: Any require_approval: Any @@ -21,6 +23,7 @@ class CallModelInput(TypedDict, total=False): allow_final_response: Any strict_final_response: bool hooks: Any + doom_loop: Any CallModelInputWithState = CallModelInput @@ -34,6 +37,8 @@ class ResolvedCallModelInput(TypedDict, total=False): _EXCLUDED = { + "active_tools", + "stream_replay", "stop_when", "state", "require_approval", @@ -46,6 +51,7 @@ class ResolvedCallModelInput(TypedDict, total=False): "allow_final_response", "strict_final_response", "hooks", + "doom_loop", } diff --git a/src/openrouter_agent/call_model.py b/src/openrouter_agent/call_model.py index daad651..7b8b1a4 100644 --- a/src/openrouter_agent/call_model.py +++ b/src/openrouter_agent/call_model.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Mapping, Optional +from typing import Any, Dict, Mapping, Optional from .hooks_resolve import resolve_hooks from .model_result import ModelResult @@ -8,10 +8,29 @@ def call_model(client: Any, request: Mapping[str, Any], options: Optional[Mapping[str, Any]] = None) -> ModelResult: + from .tool_set import strip_tool_set_snapshot_metadata + from .tool_types import is_server_tool + tools = request.get("tools") - final_request = dict(request) + active_tools = request.get("active_tools") + # Narrow tools to the active subset (if provided) before API conversion + # and before they are registered for execution, so the model cannot call + # filtered tools and the executor does not carry orphaned definitions. + # Server tools always pass the filter. + if active_tools is not None and tools is not None: + active_set = set(active_tools) + tools = [t for t in tools if is_server_tool(t) or t.get("function", {}).get("name") in active_set] + # Collapse a filtered-to-empty (or explicitly empty) tools list to None so + # the outbound `tools` key is omitted entirely instead of sending + # `tools: []` -- several providers reject an empty array outright. + if tools is not None and len(tools) == 0: + tools = None + final_request: Dict[Any, Any] = dict(request) + strip_tool_set_snapshot_metadata(final_request) for key in ( "tools", + "active_tools", + "stream_replay", "stop_when", "state", "require_approval", @@ -24,6 +43,7 @@ def call_model(client: Any, request: Mapping[str, Any], options: Optional[Mappin "allow_final_response", "strict_final_response", "hooks", + "doom_loop", ): final_request.pop(key, None) if tools is not None: @@ -50,5 +70,6 @@ def call_model(client: Any, request: Mapping[str, Any], options: Optional[Mappin "allow_final_response": request.get("allow_final_response"), "strict_final_response": request.get("strict_final_response"), "hooks": resolve_hooks(request.get("hooks")), + "doom_loop": request.get("doom_loop"), } ) diff --git a/src/openrouter_agent/chat_compat.py b/src/openrouter_agent/chat_compat.py index 8861494..4360c8a 100644 --- a/src/openrouter_agent/chat_compat.py +++ b/src/openrouter_agent/chat_compat.py @@ -6,6 +6,11 @@ from ._utils import get_field from .stream_transformers import extract_responses_message_from_response +ChatMessageInputItem = Dict[str, Any] +"""An OpenResponses input item emitted by `from_chat_messages`: a message item +(user / system / assistant / developer), a ``function_call`` item, or a +``function_call_output`` item.""" + def _content_to_string(content: Any) -> str: if content is None: @@ -15,8 +20,18 @@ def _content_to_string(content: Any) -> str: return json.dumps(content, separators=(",", ":")) -def from_chat_messages(messages: Sequence[Mapping[str, Any]]) -> List[Dict[str, Any]]: - output: List[Dict[str, Any]] = [] +def _tool_call_arguments(arguments: Any) -> str: + # Chat-format arguments are already a JSON string and are forwarded as-is + # (never re-stringified). A pre-parsed mapping is serialized once. + if arguments is None: + return "{}" + if isinstance(arguments, str): + return arguments + return json.dumps(arguments, separators=(",", ":")) + + +def from_chat_messages(messages: Sequence[Mapping[str, Any]]) -> List[ChatMessageInputItem]: + output: List[ChatMessageInputItem] = [] for msg in messages: role = msg.get("role") if role == "tool": @@ -27,8 +42,36 @@ def from_chat_messages(messages: Sequence[Mapping[str, Any]]) -> List[Dict[str, "output": _content_to_string(msg.get("content")), } ) - else: - output.append({"type": "message", "role": role, "content": _content_to_string(msg.get("content"))}) + continue + + content = _content_to_string(msg.get("content")) + if role == "assistant": + tool_calls = list(msg.get("tool_calls") or msg.get("toolCalls") or []) + + # Skip the message item only when there is no content AND there are + # tool calls to emit in its place. A content-less assistant message + # with no tool calls still round-trips as an empty message. + if content or not tool_calls: + output.append({"type": "message", "role": role, "content": content}) + + # One function_call item per tool call. + for tool_call in tool_calls: + call_id = get_field(tool_call, "id") + function = get_field(tool_call, "function", {}) or {} + output.append( + { + "type": "function_call", + "callId": call_id, + "id": call_id, + "name": get_field(function, "name"), + "arguments": _tool_call_arguments(get_field(function, "arguments")), + "status": "completed", + } + ) + continue + + # System, user, developer messages + output.append({"type": "message", "role": role, "content": content}) return output diff --git a/src/openrouter_agent/claude_type_guards.py b/src/openrouter_agent/claude_type_guards.py index 3cd1ddb..7c87ae6 100644 --- a/src/openrouter_agent/claude_type_guards.py +++ b/src/openrouter_agent/claude_type_guards.py @@ -1,9 +1,83 @@ +"""Ports `packages/agent/src/lib/claude-type-guards.ts`.""" + from __future__ import annotations from typing import Any, Mapping, Sequence +from .claude_constants import ClaudeContentBlockType, NonClaudeMessageRole + +_NON_CLAUDE_ROLES = frozenset(role.value for role in NonClaudeMessageRole) + + +def _is_record(value: Any) -> bool: + return isinstance(value, Mapping) + + +def _is_non_claude_role(role: Any) -> bool: + return isinstance(role, str) and role in _NON_CLAUDE_ROLES + + +def _is_claude_tool_result_block(block: Any) -> bool: + return _is_record(block) and block.get("type") == ClaudeContentBlockType.ToolResult.value + + +def _is_claude_image_block_with_source(block: Any) -> bool: + return ( + _is_record(block) + and block.get("type") == ClaudeContentBlockType.Image.value + and "source" in block + and _is_record(block["source"]) + ) + + +def _is_claude_tool_use_block_with_id(block: Any) -> bool: + return ( + _is_record(block) + and block.get("type") == ClaudeContentBlockType.ToolUse.value + and isinstance(block.get("id"), str) + ) + + +def _has_claude_specific_blocks(content: Sequence[Any]) -> bool: + return any( + _is_claude_tool_result_block(block) + or _is_claude_image_block_with_source(block) + or _is_claude_tool_use_block_with_id(block) + for block in content + ) + + +def _is_array(value: Any) -> bool: + return isinstance(value, (list, tuple)) + def is_claude_style_messages(value: Any) -> bool: - if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + """Whether ``value`` is in Claude message format. + + Structural analysis, matching upstream: plain ``{role, content: str}`` + messages are ambiguous (also valid chat/Responses input) and do NOT count. + Returns True only once a message carries a Claude-specific content block + (``tool_result``, ``image`` with a ``source`` object, or ``tool_use`` with a + string ``id``). Any ``system`` / ``developer`` / ``tool`` role seen before + that makes it non-Claude. Non-mapping entries, role-less entries, and + entries with a top-level ``type`` (Responses items) are skipped. + """ + if not _is_array(value) or len(value) == 0: return False - return all(isinstance(item, Mapping) and item.get("role") in {"user", "assistant"} for item in value) + + for message in value: + if not _is_record(message): + continue + if "role" not in message: + continue + if "type" in message: + continue # Claude messages don't have a top-level "type" + + if _is_non_claude_role(message["role"]): + return False + + content = message.get("content") + if _is_array(content) and _has_claude_specific_blocks(content): + return True + + return False diff --git a/src/openrouter_agent/conversation_state.py b/src/openrouter_agent/conversation_state.py index 7d1bf0e..eb35d42 100644 --- a/src/openrouter_agent/conversation_state.py +++ b/src/openrouter_agent/conversation_state.py @@ -5,9 +5,11 @@ import time import uuid from dataclasses import replace -from typing import Any, Dict, List, Mapping, Optional, Sequence +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple -from ._utils import dump, json_dumps, maybe_await +from pydantic import BaseModel + +from ._utils import dump, json_dumps, maybe_await, validate_schema from .tool_types import ( ConversationState, ParsedToolCall, @@ -99,6 +101,12 @@ def serialize_conversation_state(state: ConversationState) -> str: """ payload = dataclasses.asdict(state) payload["version"] = state.version if state.version is not None else CONVERSATION_STATE_VERSION + # Upstream writes `preToolUseApplied` only when true (it is `?: true`), so + # omit the default marker to keep unprepared pending calls in the legacy + # wire shape. + for item in payload.get("pending_tool_calls") or []: + if isinstance(item, dict) and item.get("pre_tool_use_applied") is not True: + item.pop("pre_tool_use_applied", None) # State built from live responses can hold SDK pydantic items (e.g. # OutputFunctionCallItem), which dataclasses.asdict passes through # untouched; dump() them to plain dicts so the wire format stays JSON. @@ -160,7 +168,16 @@ def deserialize_conversation_state(raw_json: str) -> ConversationState: pending_tool_calls = None if parsed.get("pending_tool_calls") is not None: - pending_tool_calls = [ParsedToolCall(**item) for item in parsed["pending_tool_calls"]] + # Legacy blobs predate `pre_tool_use_applied`; absence (or any value + # other than `true`, as upstream's `=== true` check) means PreToolUse + # has not run for the call yet. + pending_tool_calls = [ + ParsedToolCall( + **{key: value for key, value in item.items() if key != "pre_tool_use_applied"}, + pre_tool_use_applied=item.get("pre_tool_use_applied") is True, + ) + for item in parsed["pending_tool_calls"] + ] unsent_tool_results = None if parsed.get("unsent_tool_results") is not None: @@ -182,6 +199,8 @@ def deserialize_conversation_state(raw_json: str) -> ConversationState: partial_response=partial_response, interrupted_by=parsed.get("interrupted_by"), version=CONVERSATION_STATE_VERSION, + consumed_forced_tool_choice_key=parsed.get("consumed_forced_tool_choice_key"), + doom_loop=parsed.get("doom_loop"), ) @@ -189,14 +208,38 @@ def append_to_messages(current: Any, new_items: Sequence[Any]) -> List[Any]: return [*normalize_input_to_array(current), *list(new_items)] +def _parse_tool_arguments(tool: Tool, arguments: Any) -> Tuple[bool, Any]: + """Parse arguments with the same schema the executor validates through, + so approval predicates and `execute` agree on defaults/coercions. + Returns `(ok, value)`; Pydantic models are dumped to plain dicts because + the predicate contract is a mapping of arguments.""" + if isinstance(arguments, str): + return False, arguments + try: + parsed = validate_schema(get_tool_function(tool).get("input_schema"), arguments) + except Exception: + return False, arguments + if isinstance(parsed, BaseModel): + parsed = parsed.model_dump() + return True, parsed + + async def tool_requires_approval( tool_call: ParsedToolCall, tools: Sequence[Tool], context: Mapping[str, Any], call_level_check: Any = None, ) -> bool: - if call_level_check is not None: - return bool(await maybe_await(call_level_check(tool_call, context))) + """Whether a tool call needs approval. + + Call-level checks always take precedence and receive a normalized copy of + the call when schema parsing succeeds, or the raw call when it does not. + Tool-level predicates receive the parsed arguments -- exactly what + `execute` will receive. A schema-invalid payload, or a parse that + succeeds with a non-mapping payload, fails closed (requires approval): a + PreToolUse hook may later repair the input, so invalid wire arguments + cannot safely bypass approval. + """ matching = next( ( candidate @@ -205,11 +248,23 @@ async def tool_requires_approval( ), None, ) + if call_level_check is not None: + if matching is None: + return bool(await maybe_await(call_level_check(tool_call, context))) + ok, parsed = _parse_tool_arguments(matching, tool_call.arguments) + if not ok: + return bool(await maybe_await(call_level_check(tool_call, context))) + return bool(await maybe_await(call_level_check(replace(tool_call, arguments=parsed), context))) if not matching: return False requirement = get_tool_function(matching).get("require_approval") if callable(requirement): - return bool(await maybe_await(requirement(tool_call.arguments, context))) + ok, parsed = _parse_tool_arguments(matching, tool_call.arguments) + if not ok: + return True + if not isinstance(parsed, Mapping): + return True + return bool(await maybe_await(requirement(parsed, context))) return bool(requirement) diff --git a/src/openrouter_agent/doom_loop.py b/src/openrouter_agent/doom_loop.py new file mode 100644 index 0000000..63373f7 --- /dev/null +++ b/src/openrouter_agent/doom_loop.py @@ -0,0 +1,1289 @@ +"""Doom-loop detection for the tool-execution loop. + +Port of upstream ``src/lib/doom-loop.ts``. + +A "doom loop" is a run that stops making progress while continuing to spend: +the model re-issues the same tool call with identical arguments (including +repeated *empty* calls and repeated unparseable calls), or emits the same text +tokens over and over. This module provides the deterministic detection +primitives; ``ModelResult`` wires them into the loop behind the ``doom_loop`` +option on ``call_model``. + +Design principles (see upstream for the full rationale): + +- **Deterministic.** Every verdict is a pure function of the recorded sequence + of tool calls / assistant texts — no wall clocks, no randomness. +- **Cross-port fingerprints.** Key material is canonicalized per RFC 8785 (JCS) + and hashed with SHA-256 over the UTF-8 bytes. The stdlib ``json.dumps`` is + NOT JCS (``json.dumps(-0.0)`` is ``-0.0``; it sorts keys by code point, not + UTF-16 code unit; it does not use ECMAScript number formatting), so this + module implements the canonicalization itself. The shared vector file + ``tests/vectors/doom-loop-fingerprints.json`` is the conformance contract. +- **Tool-declared identity.** A tool opts into precise loop identity via + ``loop_key``: a function computing key material, a field-name list, or + ``False`` to exempt the tool. +- **Round-scoped streaks.** A streak measures the model *re-issuing a call + after seeing its result*. N identical calls fanned out in ONE round count + once. A round's identity for one tool is the *set* of fingerprints it was + called with; declare the complete set before scoring any of its calls (see + :meth:`DoomLoopMonitor.declare_round`). +- **Graduated response.** Detection feeds a configurable action ladder + (observe → steer → escalate → block → stop). +- **Never the cause of failure.** Any internal error degrades to a fallback + identity or skips detection — it never rejects the call or crashes the run. + +Python-specific notes (deliberate divergences): + +- Python has no ``undefined``. A ``loop_key`` function returning ``None`` is + treated like upstream's ``null`` (exempt this call); there is no "returned + undefined" fallback. ``None`` as the *declaration* means "absent" (full + arguments), matching upstream's ``undefined`` declaration. +- Python ``int`` is arbitrary precision. JSON numbers are IEEE-754 doubles in + JS, so integers are canonicalized through ``float`` (exactly as + ``JSON.parse`` would have rounded them) to keep fingerprints identical across + ports; integers outside the double range are rejected like non-finite + numbers. There is no ``bigint`` analog. +- Callable values are dropped from objects and become ``null`` in arrays + (upstream's function/``undefined`` semantics). Pydantic models are dumped in + JSON mode first. Other non-JSON types raise ``TypeError``. +- Persisted-state and verdict keys are snake_case (``round_fingerprints``, + ``call_streaks``, ``escalations_used``, ``stop_verdict``, ``pending_steer``, + ``tool_name``), matching the rest of this port's ``ConversationState``. + :meth:`DoomLoopMonitor.restore` additionally accepts upstream's camelCase + keys so a blob written by the TypeScript SDK restores too. +- Warnings use :func:`warnings.warn` (upstream: ``console.warn``). +""" + +from __future__ import annotations + +import hashlib +import math +import re +import warnings +from dataclasses import dataclass, field +from decimal import Decimal +from types import MappingProxyType +from typing import Any, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Union, cast, overload + +from typing_extensions import Literal, NotRequired, TypeAlias, TypedDict + +__all__ = [ + "DEFAULT_DOOM_LOOP_LADDER", + "DEFAULT_MAX_ESCALATIONS", + "MAX_CANONICALIZE_DEPTH", + "DoomLoopAction", + "DoomLoopCallRecord", + "DoomLoopConfig", + "DoomLoopDetectorKind", + "DoomLoopEscalationConfig", + "DoomLoopLadder", + "DoomLoopMonitor", + "DoomLoopOption", + "DoomLoopRoundCall", + "DoomLoopSerializedState", + "DoomLoopStreak", + "DoomLoopTextOptions", + "DoomLoopVerdict", + "LoopKeyResolution", + "ResolvedDoomLoopConfig", + "ResolvedEscalationConfig", + "TextRepetitionResult", + "canonicalize_key_material", + "detect_text_repetition", + "fingerprint_key_material", + "fingerprint_tool_call", + "resolve_doom_loop_option", + "resolve_ladder_action", + "resolve_loop_key_material", +] + +# region Public Types + +#: Escalation actions, weakest to strongest. See upstream docs for semantics. +DoomLoopAction: TypeAlias = Literal["observe", "steer", "escalate", "block", "stop"] + +#: Which detector produced a verdict. +DoomLoopDetectorKind: TypeAlias = Literal[ + "tool-fingerprint", + "server-tool-fingerprint", + "text-repetition", + "text-streak", +] + + +class DoomLoopVerdict(TypedDict): + """A doom-loop detection event. + + ``streak`` is the repetition count that crossed a ladder threshold. + ``tool_name`` is present for fingerprint verdicts only. ``message`` is used + verbatim for block outputs and steer messages. + """ + + detector: DoomLoopDetectorKind + action: DoomLoopAction + streak: int + fingerprint: str + tool_name: NotRequired[str] + message: str + + +class DoomLoopCallRecord(TypedDict): + """Result of recording one tool call with the monitor. + + ``duplicate_in_round`` is True when the same ``(tool_name, fingerprint)`` + was already recorded in this round: the streak did NOT increment and the + caller should reuse the decision it applied for the first occurrence. + ``verdict`` is present only when a ladder rung fired. + """ + + fingerprint: str + streak: int + duplicate_in_round: bool + verdict: NotRequired[DoomLoopVerdict] + + +class DoomLoopLadder(TypedDict, total=False): + """Streak thresholds per action; ``False`` disables a rung. Strongest wins.""" + + observe: Union[int, Literal[False]] + steer: Union[int, Literal[False]] + escalate: Union[int, Literal[False]] + block: Union[int, Literal[False]] + stop: Union[int, Literal[False]] + + +class DoomLoopEscalationConfig(TypedDict, total=False): + """Recovery configuration for the ``escalate`` ladder action.""" + + #: Model slug to run the NEXT turn on (one request only). + model: str + #: Force an ``openrouter:advisor`` consult on the next turn. ``True`` uses + #: defaults; a mapping is passed through as the advisor tool's parameters. + advisor: Union[bool, Dict[str, Any]] + #: Cap on escalations per run. Default :data:`DEFAULT_MAX_ESCALATIONS`. + max_escalations: int + + +class DoomLoopTextOptions(TypedDict, total=False): + """Options for text-loop detection.""" + + #: Master switch for both text detectors. Default True. + enabled: bool + #: Longest repeating token block to search for. Default 16. + max_period_tokens: int + #: Minimum consecutive repeats of a block to count as repetition. Default 4. + min_repeats: int + #: Minimum total tokens covered by the repetition. Default 12. + min_covered_tokens: int + #: Only the trailing N tokens of a response are scanned. Default 400. + max_window_tokens: int + + +class DoomLoopConfig(TypedDict, total=False): + """Configuration for doom-loop detection. ``doom_loop=True`` uses all defaults.""" + + ladder: DoomLoopLadder + text: Union[bool, DoomLoopTextOptions] + escalation: DoomLoopEscalationConfig + + +#: The ``doom_loop`` option on ``call_model``: ``True`` for defaults, or a config. +DoomLoopOption: TypeAlias = Union[bool, DoomLoopConfig] + + +class DoomLoopStreak(TypedDict): + """Consecutive-repetition counter for one identity (a tool, or step text). + + ``round_fingerprints``: the full fingerprint set of the tool's last round, + present only when that round had more than one distinct call. + ``call_streaks``: per-call streaks for the tool's last round (fingerprint → + consecutive rounds), omitted when reconstructible from the set + streak. + Neither is ever present on text streaks. + """ + + fingerprint: str + streak: int + round_fingerprints: NotRequired[List[str]] + call_streaks: NotRequired[Dict[str, int]] + + +class DoomLoopSerializedState(TypedDict, total=False): + """Plain-JSON detector state, persisted inside ``ConversationState.doom_loop``. + + ``tools`` and ``text`` / ``escalations_used`` are produced by + :meth:`DoomLoopMonitor.get_state`. ``stop_verdict`` and ``pending_steer`` + are engine-owned: the engine merges them into the persisted blob and + restores them itself (the monitor ignores them). + """ + + tools: Dict[str, DoomLoopStreak] + text: DoomLoopStreak + stop_verdict: DoomLoopVerdict + pending_steer: List[str] + escalations_used: int + + +class _ExemptResolution(TypedDict): + kind: Literal["exempt"] + + +class _KeyResolution(TypedDict): + kind: Literal["key"] + key_material: Any + + +class _FallbackResolution(TypedDict): + kind: Literal["fallback"] + key_material: Any + warning: str + + +#: The result of resolving a tool's ``loop_key`` declaration against one call's +#: arguments. See :func:`resolve_loop_key_material`. +LoopKeyResolution: TypeAlias = Union[_ExemptResolution, _KeyResolution, _FallbackResolution] + + +class DoomLoopRoundCall(TypedDict): + """One entry of :meth:`DoomLoopMonitor.declare_round` (upstream: inline type).""" + + tool_name: str + key_material: Any + + +class TextRepetitionResult(TypedDict): + """A detected repeating token block at the tail of a response.""" + + #: Consecutive repeats of the block (feeds the ladder as the streak). + repeats: int + #: Block length in whitespace-delimited tokens. + period_tokens: int + #: repeats × period_tokens. + covered_tokens: int + #: The repeating block itself. + sample: str + + +class ResolvedEscalationConfig(TypedDict): + """Escalation config with the budget resolved.""" + + model: NotRequired[str] + advisor: NotRequired[Union[bool, Dict[str, Any]]] + max_escalations: int + + +class ResolvedDoomLoopConfig(TypedDict): + """Fully-resolved config used by the monitor (every ladder/text key present).""" + + ladder: DoomLoopLadder + text: DoomLoopTextOptions + escalation: Optional[ResolvedEscalationConfig] + + +# endregion + +# region Defaults & Config Resolution + +#: Default ladder: observe at 2 consecutive identical rounds, block at 3, stop +#: at 6. Steer and escalate are off by default. Read-only. +DEFAULT_DOOM_LOOP_LADDER: Mapping[str, Union[int, Literal[False]]] = MappingProxyType( + { + "observe": 2, + "steer": False, + "escalate": False, + "block": 3, + "stop": 6, + } +) + +#: Default escalation budget when the rung is enabled without a cap. +DEFAULT_MAX_ESCALATIONS = 2 + +_DEFAULT_TEXT_OPTIONS: Mapping[str, Any] = MappingProxyType( + { + "enabled": True, + "max_period_tokens": 16, + "min_repeats": 4, + "min_covered_tokens": 12, + "max_window_tokens": 400, + } +) + +#: Ladder rungs ordered weakest → strongest. +_LADDER_ORDER: Tuple[str, ...] = ("observe", "steer", "escalate", "block", "stop") + + +def _is_number(value: object) -> bool: + """``typeof value === 'number'`` — bool is NOT a number here.""" + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _is_finite_number(value: object) -> bool: + if not _is_number(value): + return False + try: + return math.isfinite(cast(float, value)) + except OverflowError: + return False + + +def _sanitize_threshold(value: object, fallback: Union[int, Literal[False]]) -> Union[int, Literal[False]]: + if value is False: + return False + if _is_finite_number(value) and cast(float, value) >= 1: + return int(math.floor(cast(float, value))) + return fallback + + +def _warn(message: str) -> None: + warnings.warn(message, stacklevel=3) + + +def _warn_on_ladder_hazards(ladder: DoomLoopLadder) -> None: + """Warn about accepted-but-probably-wrong ladders (block without stop; dead rungs).""" + if ladder["block"] is not False and ladder["stop"] is False: + _warn( + "[DoomLoop] ladder has block enabled with stop disabled: a model that keeps " + "re-issuing a blocked call loops indefinitely (each blocked round still costs a " + "model request). Bound the run with stop_when, or enable the stop rung." + ) + rungs = cast(Mapping[str, Union[int, Literal[False]]], ladder) + for weak_index, weak_name in enumerate(_LADDER_ORDER): + weak_threshold = rungs[weak_name] + if weak_threshold is False: + continue + for strong_name in _LADDER_ORDER[weak_index + 1 :]: + strong_threshold = rungs[strong_name] + if strong_threshold is not False and weak_threshold >= strong_threshold: + _warn( + f'[DoomLoop] ladder rung "{weak_name}" ({weak_threshold}) can never fire: ' + f'stronger rung "{strong_name}" ({strong_threshold}) already wins at that streak ' + "(strongest crossed rung is applied)." + ) + break # one warning per dead rung is enough + + +@overload +def resolve_doom_loop_option(option: Union[DoomLoopConfig, Literal[True]]) -> ResolvedDoomLoopConfig: ... + + +@overload +def resolve_doom_loop_option(option: Optional[DoomLoopOption]) -> Optional[ResolvedDoomLoopConfig]: ... + + +def resolve_doom_loop_option(option: Optional[DoomLoopOption]) -> Optional[ResolvedDoomLoopConfig]: + """Normalize the ``doom_loop`` option. ``None`` / ``False`` → ``None`` (off). + + ``True`` or a config mapping always yields a resolved config, so + ``DoomLoopMonitor(resolve_doom_loop_option(True))`` is well-typed. + """ + if option is None or option is False: + return None + config: Mapping[str, Any] = {} if option is True else cast(Mapping[str, Any], option) + ladder_input: Mapping[str, Any] = config.get("ladder") or {} + text_raw = config.get("text") + text_input: Union[Literal[False], Mapping[str, Any]] = ( + {} if text_raw is None or text_raw is True else cast(Union[Literal[False], Mapping[str, Any]], text_raw) + ) + ladder: DoomLoopLadder = { + "observe": _sanitize_threshold(ladder_input.get("observe"), DEFAULT_DOOM_LOOP_LADDER["observe"]), + "steer": _sanitize_threshold(ladder_input.get("steer"), DEFAULT_DOOM_LOOP_LADDER["steer"]), + "escalate": _sanitize_threshold(ladder_input.get("escalate"), DEFAULT_DOOM_LOOP_LADDER["escalate"]), + "block": _sanitize_threshold(ladder_input.get("block"), DEFAULT_DOOM_LOOP_LADDER["block"]), + "stop": _sanitize_threshold(ladder_input.get("stop"), DEFAULT_DOOM_LOOP_LADDER["stop"]), + } + _warn_on_ladder_hazards(ladder) + + # Escalation recovery: usable only when the config names at least one + # mechanism. `advisor: False` is an explicit opt-out, not a mechanism — + # `{"advisor": False}` alone must read the same as `{}`. + escalation_input = cast(Optional[Mapping[str, Any]], config.get("escalation")) + advisor = escalation_input.get("advisor") if escalation_input is not None else None + has_advisor = advisor is not None and advisor is not False + model = escalation_input.get("model") if escalation_input is not None else None + has_mechanism = escalation_input is not None and (model is not None or has_advisor) + escalation: Optional[ResolvedEscalationConfig] = None + if has_mechanism: + assert escalation_input is not None + cap = escalation_input.get("max_escalations") + escalation = { + "max_escalations": int(math.floor(cast(float, cap))) + if _is_finite_number(cap) and cast(float, cap) >= 1 + else DEFAULT_MAX_ESCALATIONS + } + if model is not None: + escalation["model"] = model + if has_advisor: + escalation["advisor"] = cast(Union[bool, Dict[str, Any]], advisor) + if ladder["escalate"] is False: + _warn( + "[DoomLoop] escalation config provided but the escalate ladder rung is disabled; " + "set ladder.escalate to a streak threshold for recovery to trigger." + ) + elif ladder["escalate"] is not False: + _warn( + "[DoomLoop] ladder.escalate is enabled but no escalation config (model/advisor) was " + "provided; the rung is skipped and verdicts fall through to weaker rungs." + ) + + text: DoomLoopTextOptions + if text_input is False: + text = cast(DoomLoopTextOptions, {**_DEFAULT_TEXT_OPTIONS, "enabled": False}) + else: + + def _pick(key: str) -> Any: + value = text_input.get(key) + return _DEFAULT_TEXT_OPTIONS[key] if value is None else value + + text = { + "enabled": _pick("enabled"), + "max_period_tokens": _pick("max_period_tokens"), + "min_repeats": _pick("min_repeats"), + "min_covered_tokens": _pick("min_covered_tokens"), + "max_window_tokens": _pick("max_window_tokens"), + } + + return {"ladder": ladder, "escalation": escalation, "text": text} + + +# endregion + +# region loop_key Resolution + + +def resolve_loop_key_material(loop_key: object, args: Mapping[str, Any]) -> LoopKeyResolution: + """Resolve a tool's ``loop_key`` declaration against one call's arguments. + + - ``None`` (absent declaration) → the full arguments. + - ``False`` → the tool is statically exempt. + - list/tuple of field names → the declarative subset. An empty list, or a + list whose every field is absent, falls back to full arguments with a + warning (it would give every call the same identity). + - callable → called with the arguments. Returning ``None`` exempts THIS + call (Python has no ``undefined``; see module docstring). A raise falls + back to the full arguments with a warning. + - anything else → fallback with a warning. + """ + if loop_key is None: + return {"kind": "key", "key_material": args} + if loop_key is False: + return {"kind": "exempt"} + if isinstance(loop_key, (list, tuple)): + if len(loop_key) == 0: + return { + "kind": "fallback", + "key_material": args, + "warning": ( + "loop_key is an empty field list, which would give every call to this tool the same " + "identity; falling back to full arguments. Use `False` to exempt the tool." + ), + } + subset: Dict[str, Any] = {} + for field_name in loop_key: + if isinstance(field_name, str) and field_name in args: + subset[field_name] = args[field_name] + if len(subset) == 0: + names = ", ".join(f for f in loop_key if isinstance(f, str)) + return { + "kind": "fallback", + "key_material": args, + "warning": ( + f"loop_key fields [{names}] are all absent from the arguments, which would give every " + "call the same identity; falling back to full arguments." + ), + } + return {"kind": "key", "key_material": subset} + if callable(loop_key): + try: + result = loop_key(args) + except Exception as error: # detection must never take down a run + return { + "kind": "fallback", + "key_material": args, + "warning": f"loop_key threw ({error}); falling back to full arguments", + } + if result is None: + return {"kind": "exempt"} + return {"kind": "key", "key_material": result} + return { + "kind": "fallback", + "key_material": args, + "warning": f"loop_key has unsupported shape ({type(loop_key).__name__}); falling back to full arguments", + } + + +# endregion + +# region Canonicalization & Fingerprinting + +#: Maximum nesting depth accepted by :func:`canonicalize_key_material`. +MAX_CANONICALIZE_DEPTH = 64 + +_SHORT_ESCAPES = { + 0x08: "\\b", + 0x09: "\\t", + 0x0A: "\\n", + 0x0C: "\\f", + 0x0D: "\\r", + 0x22: '\\"', + 0x5C: "\\\\", +} + + +def _quote_json_string(value: str) -> str: + """ECMAScript ``JSON.stringify`` string quoting (the JCS string form). + + Escapes ``"``, ``\\``, the short control escapes, other C0 controls as + lowercase ``\\u00xx``, and LONE surrogates as lowercase ``\\udxxx``. A + surrogate PAIR stored as two code points is a valid pair in UTF-16 (what + JS sees), so it is combined and emitted raw. Everything else is raw. + """ + out: List[str] = ['"'] + i = 0 + length = len(value) + while i < length: + code = ord(value[i]) + if code in _SHORT_ESCAPES: + out.append(_SHORT_ESCAPES[code]) + elif code < 0x20: + out.append(f"\\u{code:04x}") + elif 0xD800 <= code <= 0xDBFF: + if i + 1 < length and 0xDC00 <= ord(value[i + 1]) <= 0xDFFF: + low = ord(value[i + 1]) + out.append(chr(0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00))) + i += 1 + else: + out.append(f"\\u{code:04x}") + elif 0xDC00 <= code <= 0xDFFF: + out.append(f"\\u{code:04x}") + else: + out.append(value[i]) + i += 1 + out.append('"') + return "".join(out) + + +def _es_number_to_string(value: float) -> str: + """ECMAScript ``Number::toString`` for a finite double (the JCS number form). + + ``repr(float)`` yields the shortest round-tripping digit string (the same + digits ECMAScript selects); this only reformats them per ES rules. + """ + if value == 0: + return "0" # covers -0 + if value < 0: + return "-" + _es_number_to_string(-value) + sign, digit_tuple, exponent = Decimal(repr(value)).as_tuple() + digits = list(digit_tuple) + exp = int(exponent) + while len(digits) > 1 and digits[-1] == 0: + digits.pop() + exp += 1 + while len(digits) > 1 and digits[0] == 0: + digits.pop(0) + text = "".join(str(d) for d in digits) + k = len(text) + n = exp + k + if k <= n <= 21: + return text + "0" * (n - k) + if 0 < n <= 21: + return text[:n] + "." + text[n:] + if -6 < n <= 0: + return "0." + "0" * (-n) + text + e = n - 1 + exp_text = ("+" if e >= 0 else "-") + str(abs(e)) + if k == 1: + return text + "e" + exp_text + return text[0] + "." + text[1:] + "e" + exp_text + + +def _utf16_sort_key(value: str) -> bytes: + return value.encode("utf-16-be", "surrogatepass") + + +def canonicalize_key_material(value: Any) -> str: + """RFC 8785 (JCS) canonical JSON for doom-loop fingerprinting. + + Object keys sorted by UTF-16 code units (recursively), arrays in order, + strings and finite numbers serialized per ECMAScript ``JSON.stringify`` + (``-0`` → ``0``, ``1e21`` → ``1e+21``, lone surrogates → ``\\udXXX``). + Insensitive to key insertion order. + + Raises ``ValueError`` on values RFC 8785 cannot represent (non-finite + numbers, integers outside the double range), circular references, and + nesting deeper than :data:`MAX_CANONICALIZE_DEPTH`; ``TypeError`` on + non-JSON types or non-string object keys. Real tool arguments (parsed JSON) + never trigger these; the engine catches and falls back. + """ + seen: Set[int] = set() + + def canon(v: Any, depth: int) -> str: + if depth > MAX_CANONICALIZE_DEPTH: + raise ValueError( + f"Cannot canonicalize key material nested deeper than {MAX_CANONICALIZE_DEPTH} levels " + "for doom-loop fingerprinting" + ) + if v is None: + return "null" + if v is True: + return "true" + if v is False: + return "false" + if isinstance(v, int): + try: + as_float = float(v) + except OverflowError: + raise ValueError( + "Cannot canonicalize non-finite number (integer outside the IEEE-754 double range) " + "for doom-loop fingerprinting: RFC 8785 has no representation" + ) from None + return _es_number_to_string(as_float) + if isinstance(v, float): + if not math.isfinite(v): + raise ValueError( + "Cannot canonicalize non-finite number (NaN/Infinity) for doom-loop fingerprinting: " + "RFC 8785 has no representation" + ) + return _es_number_to_string(v) + if isinstance(v, str): + return _quote_json_string(v) + if _is_pydantic_model(v): + return canon(v.model_dump(mode="json"), depth) + if isinstance(v, (list, tuple, Mapping)): + marker = id(v) + if marker in seen: + raise ValueError("Cannot canonicalize circular key material for doom-loop fingerprinting") + seen.add(marker) + try: + if isinstance(v, (list, tuple)): + return "[" + ",".join(canon(item, depth + 1) for item in v) + "]" + entries: List[Tuple[str, Any]] = [] + for key, entry in v.items(): + if not isinstance(key, str): + raise TypeError( + f"Cannot canonicalize object key of type {type(key).__name__} for doom-loop " + "fingerprinting: JSON object keys are strings" + ) + if callable(entry) and not _is_pydantic_model(entry): + continue + entries.append((key, entry)) + entries.sort(key=lambda pair: _utf16_sort_key(pair[0])) + return ( + "{" + + ",".join(f"{_quote_json_string(key)}:{canon(entry, depth + 1)}" for key, entry in entries) + + "}" + ) + finally: + seen.discard(marker) + if callable(v): + # function — not representable; JSON semantics say null here. + return "null" + raise TypeError( + f"Cannot canonicalize value of type {type(v).__name__} for doom-loop fingerprinting: " + "RFC 8785 has no representation (convert it in loop_key)" + ) + + return canon(value, 0) + + +def _is_pydantic_model(value: object) -> bool: + try: + from pydantic import BaseModel + except ImportError: # pragma: no cover - pydantic is a hard dependency + return False + return isinstance(value, BaseModel) + + +def _sha256_hex(canonical: str) -> str: + # Lone surrogates never reach the encoder: _quote_json_string escapes them. + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +async def fingerprint_key_material(key_material: Any) -> str: + """Fingerprint arbitrary key material: ``sha256(utf8(jcs(key_material)))``, lowercase hex.""" + return _sha256_hex(canonicalize_key_material(key_material)) + + +async def fingerprint_tool_call(tool_name: str, key_material: Any) -> str: + """Fingerprint a tool call: ``sha256(utf8(tool_name + "\\n" + jcs(key_material)))``. + + This exact construction is the cross-port contract — see + ``tests/vectors/doom-loop-fingerprints.json``. + """ + return _sha256_hex(_tool_call_canonical(tool_name, key_material)) + + +def _tool_call_canonical(tool_name: str, key_material: Any) -> str: + return f"{tool_name}\n{canonicalize_key_material(key_material)}" + + +# endregion + +# region Text Repetition Detection + +# ECMAScript `\s` (WhiteSpace + LineTerminator). Python's `\s` is a different +# set (it includes U+001C..U+001F and excludes U+FEFF), and the normalized text +# is fingerprinted, so the class is spelled out for cross-port parity. +_JS_WHITESPACE = "\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff" +_JS_WS_RUN = re.compile(f"[{_JS_WHITESPACE}]+") +_JS_TRIM = re.compile(f"^[{_JS_WHITESPACE}]+|[{_JS_WHITESPACE}]+$") + + +def _js_trim(text: str) -> str: + return _JS_TRIM.sub("", text) + + +def _utf16_tail(text: str, budget: int) -> str: + """``text.slice(-budget)`` measured in UTF-16 code units, like JS.""" + if len(text) * 2 <= budget: + return text + encoded = text.encode("utf-16-le", "surrogatepass") + if len(encoded) // 2 <= budget: + return text + return encoded[-budget * 2 :].decode("utf-16-le", "surrogatepass") + + +def _option(options: Optional[Mapping[str, Any]], key: str) -> Any: + value = options.get(key) if options is not None else None + return _DEFAULT_TEXT_OPTIONS[key] if value is None else value + + +def detect_text_repetition( + text: str, + options: Optional[DoomLoopTextOptions] = None, +) -> Optional[TextRepetitionResult]: + """Detect a period-p token block repeating at the *tail* of ``text``. + + Whitespace tokenization, suffix comparison, no heuristics beyond the + thresholds. Only the trailing ``max_window_tokens * 64`` characters are + tokenized. Ties on covered tokens prefer the smallest period. Returns + ``None`` when no block meets ``min_repeats`` and ``min_covered_tokens``. + + KNOWN LIMIT: token blocks must repeat *exactly*. + """ + opts = cast(Optional[Mapping[str, Any]], options) + max_period_tokens = int(_option(opts, "max_period_tokens")) + min_repeats = int(_option(opts, "min_repeats")) + min_covered_tokens = int(_option(opts, "min_covered_tokens")) + max_window_tokens = int(_option(opts, "max_window_tokens")) + + char_budget = max_window_tokens * 64 + tail = _utf16_tail(text, char_budget) + all_tokens = [token for token in _JS_WS_RUN.split(_js_trim(tail)) if token] + tokens = all_tokens[-max_window_tokens:] if max_window_tokens > 0 else [] + if len(tokens) < max(min_covered_tokens, 2): + return None + + best: Optional[TextRepetitionResult] = None + count = len(tokens) + max_period = min(max_period_tokens, count // 2) + for period in range(1, max_period + 1): + repeats = 1 + tail_block = tokens[count - period :] + start = count - 2 * period + while start >= 0: + if tokens[start : start + period] != tail_block: + break + repeats += 1 + start -= period + covered_tokens = repeats * period + if repeats >= min_repeats and covered_tokens >= min_covered_tokens: + if best is None or covered_tokens > best["covered_tokens"]: + best = { + "repeats": repeats, + "period_tokens": period, + "covered_tokens": covered_tokens, + "sample": " ".join(tail_block), + } + return best + + +# endregion + +# region Ladder Resolution + + +def resolve_ladder_action( + ladder: DoomLoopLadder, + streak: float, + *, + allow_block: bool, + allow_escalate: bool = True, +) -> Optional[DoomLoopAction]: + """Map a streak onto the strongest crossed ladder rung. + + ``allow_block=False`` (text and server-tool verdicts) lets a block-level + streak fall through to escalate/steer/observe, while stop still stops. + ``allow_escalate=False`` disables the escalate rung (no mechanism, or the + budget is exhausted). A missing rung key is treated as disabled. + """ + rungs = cast(Mapping[str, Any], ladder) + + def meets(name: str) -> bool: + threshold = rungs.get(name, False) + return threshold is not False and threshold is not None and streak >= threshold + + if meets("stop"): + return "stop" + if allow_block and meets("block"): + return "block" + if allow_escalate and meets("escalate"): + return "escalate" + if meets("steer"): + return "steer" + if meets("observe"): + return "observe" + return None + + +# endregion + +# region Monitor + + +@dataclass +class _StreakEntry: + """In-memory streak entry: serialized shape + run-local round bookkeeping.""" + + fingerprint: str + streak: int + #: The round's complete declared fingerprint set, sorted. + round_fingerprints: Optional[List[str]] = None + #: Current round's per-call streaks (fingerprint → consecutive rounds). + call_streaks: Optional[Dict[str, int]] = None + #: Round of the most recent record. Never serialized. + round: Optional[int] = None + #: Fingerprints of `round` already recorded (in-round duplicate collapse). + seen_this_round: Set[str] = field(default_factory=set) + #: The previous round's set and streak, fixed at the round transition. + prior_round_fingerprints: Optional[List[str]] = None + prior_streak: int = 0 + #: The previous round's per-call streaks, fixed at the round transition. + prior_call_streaks: Dict[str, int] = field(default_factory=dict) + + +def _get_key(mapping: Mapping[str, Any], snake: str, camel: str) -> Any: + """Read a persisted field, accepting upstream's camelCase spelling too.""" + if snake in mapping: + return mapping[snake] + return mapping.get(camel) + + +def _is_valid_streak(value: object) -> bool: + return ( + isinstance(value, Mapping) + and isinstance(value.get("fingerprint"), str) + and _is_finite_number(value.get("streak")) + ) + + +def _call_streaks_reconstructible(entry: _StreakEntry) -> bool: + """True when per-call counts carry nothing beyond the round set + streak.""" + counts = entry.call_streaks + if counts is None: + return True + members = entry.round_fingerprints if entry.round_fingerprints is not None else [entry.fingerprint] + return len(counts) == len(members) and all(key in members and counts[key] == entry.streak for key in counts) + + +def _restore_streak_entry(entry: Mapping[str, Any]) -> _StreakEntry: + """Rebuild one tool's in-memory streak entry from its persisted shape.""" + fingerprint = cast(str, entry["fingerprint"]) + raw_set = _get_key(entry, "round_fingerprints", "roundFingerprints") + persisted_set: List[str] = ( + sorted(raw_set) + if isinstance(raw_set, list) and len(raw_set) > 1 and all(isinstance(v, str) for v in raw_set) + else [fingerprint] + ) + persisted_call_streaks: Dict[str, int] = {} + raw_counts = _get_key(entry, "call_streaks", "callStreaks") + if isinstance(raw_counts, Mapping): + for call_fingerprint, count in raw_counts.items(): + if isinstance(call_fingerprint, str) and _is_finite_number(count) and count >= 1: + persisted_call_streaks[call_fingerprint] = int(math.floor(count)) + # Clamp a corrupt streak to a sane count rather than dropping the entry. + streak = min(max(int(math.floor(cast(float, entry["streak"]))), 1), 1_000_000) + return _StreakEntry( + fingerprint=fingerprint, + streak=streak, + round_fingerprints=persisted_set, + call_streaks=( + persisted_call_streaks if persisted_call_streaks else {member: streak for member in persisted_set} + ), + ) + + +_ACTION_STRENGTH: Mapping[str, int] = MappingProxyType({"observe": 0, "steer": 1, "escalate": 2, "block": 3, "stop": 4}) + + +def _stronger_verdict(a: Optional[DoomLoopVerdict], b: Optional[DoomLoopVerdict]) -> Optional[DoomLoopVerdict]: + if a is None: + return b + if b is None: + return a + # Within-response wins ties: its message names the repeated block. + return b if _ACTION_STRENGTH[b["action"]] > _ACTION_STRENGTH[a["action"]] else a + + +def _summarize_round(fingerprints: Sequence[str]) -> str: + shown = [value[:8] for value in fingerprints[:3]] + if len(fingerprints) > 3: + return f"{'+'.join(shown)}+{len(fingerprints) - 3} more…" + return f"{'+'.join(shown)}…" + + +def _build_tool_verdict_message( + *, + tool_name: str, + fingerprint: str, + call_set: Sequence[str], + round_streak: int, + call_streak: int, + round_declared: bool, +) -> str: + """Verdict text for a tool-fingerprint detection (see upstream for the dedupe rationale).""" + if call_streak > round_streak or not round_declared: + return ( + f'Doom loop suspected: this exact "{tool_name}" call has been repeated across ' + "consecutive rounds. Repeating it will not change the result. " + "Take a different approach, or explain why repetition is required." + ) + if len(call_set) > 1: + return ( + f'Doom loop suspected: tool "{tool_name}" was invoked in {round_streak} consecutive rounds ' + f"with the same set of {len(call_set)} parallel calls " + f"(round identity {_summarize_round(call_set)}). Reissuing the same fan-out " + "will not change the results. Take a different approach, or explain why repetition is required." + ) + return ( + f'Doom loop suspected: tool "{tool_name}" was invoked in {round_streak} consecutive rounds ' + f"with identical arguments (fingerprint {fingerprint[:16]}…). Repeating the call " + "will not change the result. Take a different approach, or explain why repetition is required." + ) + + +class DoomLoopMonitor: + """Pure state machine over the recorded transcript. + + Feed it tool calls and assistant texts, get verdicts back. Holds no engine + concerns (hook emission, blocking, steering, stopping live in + ``ModelResult``). State is bounded plain JSON and round-trips through + ``ConversationState.doom_loop`` via :meth:`get_state` / :meth:`restore`. + """ + + def __init__(self, config: ResolvedDoomLoopConfig, initial_state: Any = None) -> None: + self._config = config + # Plain dict keyed by tool name: Python dict keys are inert data, so a + # tool named "__proto__" (or any hostile persisted name) just works. + self._tools: Dict[str, _StreakEntry] = {} + self._text: Optional[_StreakEntry] = None + self._escalations_used = 0 + self._declared_round: Optional[Tuple[int, Dict[str, List[str]]]] = None + # Per-object fingerprint memo (upstream: WeakMap). Python dicts/lists + # are not weak-referenceable, so the memo holds a strong reference + # (keeping `id()` stable) and is scoped to ONE round — it is cleared + # whenever a different round is declared or recorded, which bounds its + # size to one round's calls. Within a round it gives upstream's + # behavior: the same key-material object handed to `declare_round` and + # then `record_tool_call` is hashed once, with declare-time identity. + self._memo: Dict[int, Tuple[object, Dict[str, str]]] = {} + self._memo_round: Optional[int] = None + if initial_state is not None: + self.restore(initial_state) + + # -- fingerprint memo ------------------------------------------------- + + def _enter_memo_round(self, round: int) -> None: + if self._memo_round != round: + self._memo.clear() + self._memo_round = round + + def _fingerprint_once(self, tool_name: str, key_material: Any) -> str: + memoizable = isinstance(key_material, (dict, list)) or ( + key_material is not None and not isinstance(key_material, (str, int, float, bool, tuple)) + ) + if memoizable: + hit = self._memo.get(id(key_material)) + if hit is not None and hit[0] is key_material: + cached = hit[1].get(tool_name) + if cached is not None: + return cached + fingerprint = _sha256_hex(_tool_call_canonical(tool_name, key_material)) + if memoizable: + slot = self._memo.get(id(key_material)) + if slot is None or slot[0] is not key_material: + slot = (key_material, {}) + self._memo[id(key_material)] = slot + slot[1][tool_name] = fingerprint + return fingerprint + + # -- rounds ----------------------------------------------------------- + + async def declare_round(self, round: int, calls: Sequence[DoomLoopRoundCall]) -> None: + """Declare the complete set of calls a round will make, before any is recorded. + + Each entry is one call's ``{"tool_name", "key_material"}``; the monitor + groups them per tool into that tool's round set. Idempotent per round, + and safe to skip (per-call detection needs no declaration). Unhashable + key material is excluded from the set with a warning. Never serialized. + """ + self._enter_memo_round(round) + collected: Dict[str, Set[str]] = {} + for call in calls: + tool_name = call["tool_name"] + try: + fingerprint = self._fingerprint_once(tool_name, call["key_material"]) + except Exception as error: # fail open per call + _warn( + f'[DoomLoop] could not fingerprint a "{tool_name}" call while declaring round ' + f"{round}; excluding it from the round set: {error}" + ) + continue + collected.setdefault(tool_name, set()).add(fingerprint) + self._declared_round = (round, {name: sorted(members) for name, members in collected.items()}) + + # -- escalation budget ------------------------------------------------ + + def can_escalate(self) -> bool: + """True when a mechanism is configured and the budget is not exhausted.""" + escalation = self._config["escalation"] + return escalation is not None and self._escalations_used < escalation["max_escalations"] + + def consume_escalation(self) -> None: + """Consume one escalation. The ENGINE calls this when it applies the recovery.""" + self._escalations_used += 1 + + # -- persistence ------------------------------------------------------ + + def get_state(self) -> DoomLoopSerializedState: + """Snapshot the serializable detector state (deep copy; round markers dropped). + + ``stop_verdict`` / ``pending_steer`` are engine-owned and merged by the + engine alongside this snapshot. + """ + tools: Dict[str, DoomLoopStreak] = {} + for name, entry in self._tools.items(): + streak: DoomLoopStreak = {"fingerprint": entry.fingerprint, "streak": entry.streak} + if entry.round_fingerprints is not None and len(entry.round_fingerprints) > 1: + streak["round_fingerprints"] = list(entry.round_fingerprints) + if entry.call_streaks is not None and not _call_streaks_reconstructible(entry): + streak["call_streaks"] = dict(entry.call_streaks) + tools[name] = streak + state: DoomLoopSerializedState = {"tools": tools} + if self._text is not None: + state["text"] = {"fingerprint": self._text.fingerprint, "streak": self._text.streak} + if self._escalations_used > 0: + state["escalations_used"] = self._escalations_used + return state + + def restore(self, state: Any) -> None: + """Restore persisted state (from ``ConversationState.doom_loop``). + + Invalid blobs are ignored with a warning. Engine-owned fields + (``stop_verdict``, ``pending_steer``) are ignored here. + """ + if not isinstance(state, Mapping): + _warn("[DoomLoop] Ignoring invalid persisted doom-loop state") + return + tools: Dict[str, _StreakEntry] = {} + raw_tools = state.get("tools") + if isinstance(raw_tools, Mapping): + for name, entry in raw_tools.items(): + if isinstance(name, str) and _is_valid_streak(entry): + tools[name] = _restore_streak_entry(entry) + self._tools = tools + raw_text = state.get("text") + self._text = ( + _StreakEntry(fingerprint=raw_text["fingerprint"], streak=raw_text["streak"]) + if isinstance(raw_text, Mapping) and _is_valid_streak(raw_text) + else None + ) + used = _get_key(state, "escalations_used", "escalationsUsed") + self._escalations_used = int(math.floor(used)) if _is_finite_number(used) and used >= 0 else 0 + + # -- tool calls ------------------------------------------------------- + + async def record_tool_call( + self, + tool_name: str, + key_material: Any, + round: int, + *, + allow_block: bool = True, + detector: Literal["tool-fingerprint", "server-tool-fingerprint"] = "tool-fingerprint", + ) -> DoomLoopCallRecord: + """Record one tool call and return the streak record plus any verdict. + + Per tool: interleaved calls to other tools do not reset a tool's + streak. Per round: the same ``(tool, fingerprint)`` recorded again in + the SAME ``round`` is a duplicate. A round's identity is the *set* of + fingerprints (declare it first via :meth:`declare_round`). The + stronger of the round-set streak and the per-call streak decides. + + ``allow_block=False`` for post-execution records (server tools). + + Raises whatever :func:`canonicalize_key_material` raises for + unhashable key material; callers must catch and fall back. + """ + self._enter_memo_round(round) + fingerprint = self._fingerprint_once(tool_name, key_material) + previous = self._tools.get(tool_name) + is_same_round = previous is not None and previous.round is not None and previous.round == round + + declared: Optional[List[str]] = None + if self._declared_round is not None and self._declared_round[0] == round: + declared = self._declared_round[1].get(tool_name) + declared_member = declared is not None and fingerprint in declared + + seen: Set[str] = previous.seen_this_round if is_same_round and previous is not None else set() + duplicate_in_round = fingerprint in seen + + prior_set: Optional[List[str]] + prior_streak: int + prior_call_streaks: Dict[str, int] + if is_same_round: + assert previous is not None + prior_set = previous.prior_round_fingerprints + prior_streak = previous.prior_streak + prior_call_streaks = previous.prior_call_streaks + elif previous is not None: + prior_set = ( + previous.round_fingerprints if previous.round_fingerprints is not None else [previous.fingerprint] + ) + prior_streak = previous.streak + prior_call_streaks = previous.call_streaks if previous.call_streaks is not None else {} + else: + prior_set = None + prior_streak = 0 + prior_call_streaks = {} + + def score(members: Sequence[str]) -> int: + return prior_streak + 1 if prior_set is not None and list(prior_set) == list(members) else 1 + + call_streak = prior_call_streaks.get(fingerprint, 0) + 1 + + call_set: List[str] = declared if declared_member and declared is not None else [fingerprint] + streak = score(call_set) + round_set: List[str] = declared if declared is not None else [fingerprint] + + if is_same_round and previous is not None and previous.call_streaks is not None: + previous.call_streaks[fingerprint] = call_streak + call_streaks = previous.call_streaks + else: + call_streaks = {fingerprint: call_streak} + seen.add(fingerprint) + + self._tools[tool_name] = _StreakEntry( + fingerprint=( + previous.fingerprint + if declared is not None and not declared_member and previous is not None + else fingerprint + ), + round=round, + seen_this_round=seen, + round_fingerprints=round_set, + streak=score(round_set), + prior_round_fingerprints=prior_set, + prior_streak=prior_streak, + prior_call_streaks=prior_call_streaks, + call_streaks=call_streaks, + ) + + effective_streak = max(streak, call_streak) + action = resolve_ladder_action( + self._config["ladder"], + effective_streak, + allow_block=allow_block, + allow_escalate=self.can_escalate(), + ) + record: DoomLoopCallRecord = { + "fingerprint": fingerprint, + "streak": effective_streak, + "duplicate_in_round": duplicate_in_round, + } + if action is None: + return record + record["verdict"] = { + "detector": detector, + "action": action, + "streak": effective_streak, + "fingerprint": fingerprint, + "tool_name": tool_name, + "message": _build_tool_verdict_message( + tool_name=tool_name, + fingerprint=fingerprint, + call_set=call_set, + round_streak=streak, + call_streak=call_streak, + round_declared=declared_member, + ), + } + return record + + # -- text ------------------------------------------------------------- + + def reset_text_streak(self) -> None: + """Clear the cross-step text streak (a late async-tool result is progress).""" + self._text = None + + async def record_assistant_text(self, text: str) -> Optional[DoomLoopVerdict]: + """Record one step's assistant text; return the strongest text verdict. + + ``text-repetition``: a token block repeating within THIS response. + ``text-streak``: identical (whitespace-normalized) text across + consecutive steps. Empty/whitespace-only text is a no-op. + """ + text_options = self._config["text"] + if not text_options.get("enabled", True): + return None + normalized = _js_trim(_JS_WS_RUN.sub(" ", text)) + if len(normalized) == 0: + return None + + within_response: Optional[DoomLoopVerdict] = None + repetition = detect_text_repetition(normalized, text_options) + if repetition is not None: + action = resolve_ladder_action( + self._config["ladder"], + repetition["repeats"], + allow_block=False, + allow_escalate=self.can_escalate(), + ) + if action is not None: + within_response = { + "detector": "text-repetition", + "action": action, + "streak": repetition["repeats"], + "fingerprint": await fingerprint_key_material(repetition["sample"]), + "message": ( + f'Doom loop suspected: the response repeats "{repetition["sample"]}" ' + f"{repetition['repeats']} times in a row. Stop repeating and take a different approach." + ), + } + + fingerprint = await fingerprint_key_material(normalized) + previous = self._text + streak = previous.streak + 1 if previous is not None and previous.fingerprint == fingerprint else 1 + self._text = _StreakEntry(fingerprint=fingerprint, streak=streak) + cross_step: Optional[DoomLoopVerdict] = None + cross_action = resolve_ladder_action( + self._config["ladder"], + streak, + allow_block=False, + allow_escalate=self.can_escalate(), + ) + if cross_action is not None: + cross_step = { + "detector": "text-streak", + "action": cross_action, + "streak": streak, + "fingerprint": fingerprint, + "message": ( + f"Doom loop suspected: the assistant produced identical text for {streak} " + "consecutive turns. Stop repeating and take a different approach." + ), + } + + return _stronger_verdict(within_response, cross_step) + + +# endregion diff --git a/src/openrouter_agent/hooks_schemas.py b/src/openrouter_agent/hooks_schemas.py index 3bccb34..f8532a9 100644 --- a/src/openrouter_agent/hooks_schemas.py +++ b/src/openrouter_agent/hooks_schemas.py @@ -31,6 +31,7 @@ class HookName(str, Enum): SessionStart = "SessionStart" SessionEnd = "SessionEnd" PostModelCall = "PostModelCall" + DoomLoopDetected = "DoomLoopDetected" @dataclass(frozen=True) @@ -70,13 +71,9 @@ class PostToolUsePayload(BaseModel): class PostToolUseFailurePayload(BaseModel): - """Fired when a tool EXECUTION throws or returns an error. - - Deliberately NOT fired when a tool never ran: a PermissionRequest 'deny', - a user rejection on approval resume, or a PreToolUse block all synthesize - a rejected result without execution, so no failure event is emitted. - Observe those outcomes via the PermissionRequest / PreToolUse hooks - themselves. + """Fired when an entered tool lifecycle fails after PreToolUse, including + a mutated-input approval denial/rejection or schema-invalid mutation. + Initial approval denials and rejections do not enter the lifecycle. """ tool_name: str @@ -116,7 +113,7 @@ class SessionUsageTotals(ModelCallUsage): class SessionEndPayload(BaseModel): - reason: Literal["user", "error", "max_turns", "complete"] + reason: Literal["user", "error", "max_turns", "complete", "doom_loop"] total_usage: Optional[SessionUsageTotals] = None @@ -130,6 +127,26 @@ class PostModelCallPayload(BaseModel): usage: Optional[ModelCallUsage] = None +class DoomLoopDetectedPayload(BaseModel): + """Fired when doom-loop detection crosses a ladder rung (see the + ``doom_loop`` option on ``call_model``).""" + + #: Which detector fired: identical client/server tool calls, or repeated text. + detector: Literal["tool-fingerprint", "server-tool-fingerprint", "text-repetition", "text-streak"] + #: The ladder action the engine resolved for this streak. + action: Literal["observe", "steer", "escalate", "block", "stop"] + #: Consecutive repetition count that crossed a ladder rung. + streak: int + #: Deterministic fingerprint of the repeated unit (call identity or text). + fingerprint: str + #: Present for tool-fingerprint verdicts. + tool_name: Optional[str] = None + #: Present for tool-fingerprint verdicts: the repeated call's arguments. + tool_input: Optional[Dict[str, Any]] = None + #: Explanation used for block outputs / steer messages. + message: str + + # --------------------------------------------------------------------------- # Result schemas # --------------------------------------------------------------------------- @@ -165,6 +182,20 @@ class UserPromptSubmitResult(BaseModel): reject: Optional[Union[bool, str]] = None +class DoomLoopDetectedResult(BaseModel): + """Override the engine's resolved action for THIS event. + + De-escalate (``"observe"`` on a would-be block) or escalate (``"stop"`` + immediately). When several handlers override, the last override wins. + Text and server-tool verdicts cannot be blocked; an ``override_action`` + of ``"block"`` on them downgrades to ``"observe"``. ``"escalate"`` is + honored only when an ``escalation`` config exists and budget remains; + otherwise it downgrades to ``"observe"``. + """ + + override_action: Optional[Literal["observe", "steer", "escalate", "block", "stop"]] = None + + # --------------------------------------------------------------------------- # Built-in hook registry # --------------------------------------------------------------------------- @@ -179,6 +210,7 @@ class UserPromptSubmitResult(BaseModel): HookName.SessionStart.value: HookDefinition(payload=SessionStartPayload, result=None), HookName.SessionEnd.value: HookDefinition(payload=SessionEndPayload, result=None), HookName.PostModelCall.value: HookDefinition(payload=PostModelCallPayload, result=None), + HookName.DoomLoopDetected.value: HookDefinition(payload=DoomLoopDetectedPayload, result=DoomLoopDetectedResult), } BUILT_IN_HOOK_NAMES = frozenset(BUILT_IN_HOOKS.keys()) diff --git a/src/openrouter_agent/hooks_types.py b/src/openrouter_agent/hooks_types.py index 953af4c..d79efcd 100644 --- a/src/openrouter_agent/hooks_types.py +++ b/src/openrouter_agent/hooks_types.py @@ -18,7 +18,11 @@ from dataclasses import dataclass, field from typing import Any, Awaitable, Callable, Dict, List, Optional, Pattern, Sequence, Union -from .hooks_schemas import HookName # noqa: F401 (re-exported for import-surface parity) +from .hooks_schemas import ( # noqa: F401 (re-exported for import-surface parity) + DoomLoopDetectedPayload, + DoomLoopDetectedResult, + HookName, +) #: Matcher for tool-scoped hooks. Filters handler invocation by tool name. ToolMatcher = Union[str, Pattern[str], Callable[[str], bool]] diff --git a/src/openrouter_agent/item_types.py b/src/openrouter_agent/item_types.py index 66e5604..5b275ab 100644 --- a/src/openrouter_agent/item_types.py +++ b/src/openrouter_agent/item_types.py @@ -12,6 +12,15 @@ FunctionProgressItem = Dict[str, Any] FunctionResultItem = Dict[str, Any] Item = Dict[str, Any] +NewAssistantMessageItem = Dict[str, Any] +"""A new assistant message for input (not yet persisted, no id). + +Distinct from `AssistantMessageItem` (the model's output message, which has an +id and structured content). Exists so caller-supplied history, e.g. the output +of `from_chat_messages` / `from_claude_messages`, is a valid `Item`.""" +NewDeveloperMessageItem = Dict[str, Any] +NewSystemMessageItem = Dict[str, Any] +"""A new system message for input (not yet persisted, no id).""" NewUserMessageItem = Dict[str, Any] ReasoningItem = Dict[str, Any] SystemMessageItem = Dict[str, Any] diff --git a/src/openrouter_agent/model_result.py b/src/openrouter_agent/model_result.py index 11016c0..9bb9ba6 100644 --- a/src/openrouter_agent/model_result.py +++ b/src/openrouter_agent/model_result.py @@ -5,10 +5,22 @@ import json import time import warnings -from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Sequence, Tuple - -from ._utils import dump, get_field, is_async_iterable, json_dumps, maybe_await, sdk_request_kwargs -from .async_params import resolve_async_functions +from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Sequence, Tuple, cast + +from pydantic import BaseModel +from typing_extensions import Literal + +from ._utils import ( + dump, + get_field, + is_async_iterable, + json_dumps, + json_loads_maybe, + maybe_await, + sdk_request_kwargs, + validate_schema, +) +from .async_params import has_async_functions, resolve_async_functions from .conversation_state import ( append_to_messages, create_initial_state, @@ -16,12 +28,22 @@ create_unsent_result, generate_conversation_id, partition_tool_calls, + tool_requires_approval, unsent_results_to_api_format, update_state, ) +from .doom_loop import ( + DoomLoopAction, + DoomLoopMonitor, + DoomLoopVerdict, + LoopKeyResolution, + ResolvedEscalationConfig, + resolve_doom_loop_option, + resolve_loop_key_material, +) from .next_turn_params import apply_next_turn_params_to_request, execute_next_turn_params_functions from .stop_conditions import is_stop_condition_met -from .stream_transformers import extract_text_from_response, extract_tool_calls_from_response +from .stream_transformers import extract_text_from_response, extract_tool_calls_from_response, response_has_tool_calls from .tool_context import ToolContextStore, resolve_context from .tool_executor import apply_on_response_received_hooks, execute_tool from .tool_types import ( @@ -98,6 +120,86 @@ def _find_latest_user_string_index(items: Sequence[Any]) -> int: return -1 +def _relax_forced_tool_choice(tool_choice: Any) -> Any: + """Relax a spent forced tool choice: `required` / a specific tool -> + `auto`; `allowed_tools` with `mode: required` -> same set with + `mode: auto`. Everything else passes through unchanged.""" + if tool_choice is None or isinstance(tool_choice, str): + return "auto" if tool_choice == "required" else tool_choice + if get_field(tool_choice, "type") == "allowed_tools": + if get_field(tool_choice, "mode") == "required": + return {**dump(tool_choice), "mode": "auto"} + return tool_choice + return "auto" + + +def _is_forced_tool_choice(tool_choice: Any) -> bool: + """Whether a caller-configured tool choice requires this turn to call a tool.""" + if tool_choice is None or isinstance(tool_choice, str): + return tool_choice == "required" + return get_field(tool_choice, "type") != "allowed_tools" or get_field(tool_choice, "mode") == "required" + + +def _tool_choice_identity(tool_choice: Any) -> str: + from .doom_loop import canonicalize_key_material + + return canonicalize_key_material(dump(tool_choice) if tool_choice is not None else None) + + +def _forced_tool_choice_key(tool_choice: Any) -> Optional[str]: + """Stable identity for a concrete forced choice; None means tools are not forced.""" + return _tool_choice_identity(tool_choice) if _is_forced_tool_choice(tool_choice) else None + + +#: Sentinel marking a tool-call id that appeared MORE THAN ONCE in one batch +#: (upstream `DUPLICATE_CALL_ID`). Ids are model-emitted and nothing enforces +#: uniqueness; a colliding id poisons the loop-key cache entry rather than let +#: the last write win, or one call's doom-loop identity aliases onto another's. +_DUPLICATE_CALL_ID = object() + +#: "No fallback identity" marker for doom-loop evaluations (upstream passes +#: `null`). A sentinel, because `None` is legitimate key material in Python. +_NO_FALLBACK = object() + +#: Output-item fields that are volatile per invocation (ids, status, timing) +#: or are the result payload itself; excluded from server-tool identity. +_SERVER_TOOL_VOLATILE_KEYS = frozenset({"id", "type", "status", "output", "result", "results", "error"}) + + +def _extract_server_tool_identity(item: Any) -> Optional[Dict[str, Any]]: + """Identity material for a server-tool output item (upstream + `extractServerToolIdentity`): every field except volatile ones and the + result payload, so identical repeated requests fingerprint identically. + Returns None when nothing identifying remains.""" + record = item if isinstance(item, Mapping) else dump(item) + if not isinstance(record, Mapping): + return None + identity: Dict[str, Any] = {} + for key, value in record.items(): + if key in _SERVER_TOOL_VOLATILE_KEYS or callable(value): + continue + identity[key] = value + return identity or None + + +#: Cap on pending call IDs listed in a paused-conversation error message. +_MAX_PENDING_IDS_IN_ERROR = 10 + + +def _describe_non_record(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "boolean" + if isinstance(value, (int, float)): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, (list, tuple)): + return "array" + return "object" + + class ModelResult: def __init__(self, options: Mapping[str, Any]) -> None: self.options = dict(options) @@ -106,7 +208,12 @@ def __init__(self, options: Mapping[str, Any]) -> None: self._events: List[Any] = [] self._tool_calls: List[ParsedToolCall] = [] self._tool_outputs: List[Any] = [] + self._call_names: Dict[str, str] = {} self._steps: List[StepResult] = [] + #: Executed tool rounds -- the step history stop conditions see + #: (upstream `allToolExecutionRounds`): one entry per round whose + #: calls actually ran, not one per model response. + self._executed_rounds: List[StepResult] = [] self._state: Any = None self._fresh_items_for_state: List[Any] = [] self._context_store: Optional[ToolContextStore] = None @@ -128,10 +235,79 @@ def __init__(self, options: Mapping[str, Any]) -> None: "cost": 0.0, "has_cost": False, } - if (self.options.get("approve_tool_calls") or self.options.get("reject_tool_calls")) and not self.options.get( - "state" - ): + # De-duplicate decisions so a call ID listed twice cannot execute an + # approval-gated tool twice, and refuse contradictory input outright: + # an ID that is both approved and rejected has no safe interpretation. + self._approved_ids: List[str] = list(dict.fromkeys(self.options.get("approve_tool_calls") or [])) + self._rejected_ids: List[str] = list(dict.fromkeys(self.options.get("reject_tool_calls") or [])) + rejected_set = set(self._rejected_ids) + conflicting = [call_id for call_id in self._approved_ids if call_id in rejected_set] + if conflicting: + raise ValueError( + f"Tool call IDs cannot be both approved and rejected: {', '.join(conflicting)}. " + "Pass each pending call ID in exactly one of approve_tool_calls or reject_tool_calls." + ) + if (self._approved_ids or self._rejected_ids) and not self.options.get("state"): raise ValueError("approve_tool_calls and reject_tool_calls require a state accessor") + # PermissionRequest 'deny' decisions parked between the approval gate + # and the tool round, which synthesizes rejected outputs for them. + self._hook_denied_calls: Dict[str, str] = {} + # Approval is idempotent per response object and tool-call occurrence: + # each call in a response is gated at most once per run. + self._completed_approval_gates: set[Tuple[int, int]] = set() + # Forced tool-choice relaxation policy (DEV-785). Only the consumed + # semantic key is persisted; dynamic choices re-resolve every turn. + self._consumed_forced_tool_choice_key: Optional[str] = None + self._configured_tool_choice: Any = None + self._configured_forced_tool_choice_key: Optional[str] = None + self._prepared_effective_tool_choice: Any = None + self._prepared_tool_choice_commit: Optional[Tuple[str, Optional[str]]] = None + self._dispatched_tool_choice_commit: Optional[Tuple[str, Optional[str]]] = None + self._dispatched_caller_tool_choice = False + self._pending_next_request: Optional[Dict[str, Any]] = None + # Doom-loop detection (None when the `doom_loop` option is off). The + # monitor is a pure, serializable state machine; this class owns the + # engine-side consequences (hook emission, block synthesis, steer + # injection, escalation, stopping the loop). + doom_loop_config = resolve_doom_loop_option(self.options.get("doom_loop")) + self._doom_loop_monitor: Optional[DoomLoopMonitor] = ( + DoomLoopMonitor(doom_loop_config) if doom_loop_config is not None else None + ) + self._doom_loop_escalation: Optional[ResolvedEscalationConfig] = ( + doom_loop_config["escalation"] if doom_loop_config is not None else None + ) + # Armed by a 'stop' verdict; checked at the loop boundaries. Persisted + # as `stop_verdict` so a condemned run stays condemned across + # decision-only resumes; cleared by a fresh conversational turn. + self._doom_loop_stop: Optional[DoomLoopVerdict] = None + # Steer guidance queued by 'steer' (and 'escalate') verdicts, injected + # as a user message before the next follow-up request. Persisted as + # `pending_steer` so guidance queued right before a pause survives. + self._pending_doom_loop_steer: List[str] = [] + # Monotonic round counter, bumped at every execution-batch boundary. + self._doom_loop_round = 0 + # Final decision per (tool, fingerprint) in the current round, so + # in-round duplicates reuse it (no second increment / hook emission). + self._doom_loop_round_decisions: Dict[str, Dict[str, Any]] = {} + # Loop-key resolutions computed while declaring the current round, + # keyed by call id, so a `loop_key` callable runs at most once per call. + self._doom_loop_round_key_material: Dict[str, Any] = {} + # Armed by an 'escalate' verdict; consumed by the next follow-up + # request as ONE-TURN overrides. Deliberately not persisted. + self._pending_doom_loop_escalation: Optional[DoomLoopVerdict] = None + # PreToolUse outcomes parked between the approval gates and execution + # (upstream `preparedToolCalls`), keyed by call id: + # - `{"type": "ready", "call": effective_call, "mutated": bool}` -- + # PreToolUse already ran; execution uses `call` and must not emit + # PreToolUse again. + # - `{"type": "blocked", "reason": str}` -- PreToolUse blocked, the + # mutated input failed validation / was denied, or the doom-loop + # checkpoint refused the call. `_run_tool_with_hooks` synthesizes + # the error output without running the tool. + self._prepared_tool_calls: Dict[str, Dict[str, Any]] = {} + # Second-phase (mutated-input) approval gates already run, so a call + # is re-gated at most once per run: `(response_key, occurrence)`. + self._completed_mutated_approval_gates: set[Tuple[Any, Any]] = set() # -- transport ----------------------------------------------------------- @@ -169,13 +345,26 @@ async def _coerce_response(self, value: Any) -> Any: raise getattr(value, "error", RuntimeError("Responses API request failed")) if is_async_iterable(value): completed = None - async for event in value: - await self._append_event(event) - typ = get_field(event, "type") - if typ in {"response.completed", "response.incomplete"}: - completed = get_field(event, "response") - if typ == "response.failed": - raise RuntimeError(str(get_field(event, "message", "Response failed"))) + iterator = value.__aiter__() + try: + async for event in iterator: + await self._append_event(event) + typ = get_field(event, "type") + if typ == "response.failed": + raise RuntimeError(str(get_field(event, "message", "Response failed"))) + if typ in {"response.completed", "response.incomplete"}: + # The terminal event ends the response: stop reading + # rather than waiting for the source to close, which + # some transports deliver late (or never). + completed = get_field(event, "response") + break + finally: + closer = getattr(iterator, "aclose", None) + if completed is not None and closer is not None: + try: + await closer() + except Exception: # noqa: BLE001 - a failing close must not fail the turn + pass if completed is None: raise RuntimeError("Stream ended without response.completed") return completed @@ -198,11 +387,98 @@ async def _coerce_response(self, value: Any) -> Any: async def _send_and_track(self, request: Mapping[str, Any], turn_type: str, turn_number: int) -> Any: """Send a request, coerce its response, and emit PostModelCall for it.""" + self._begin_tool_choice_dispatch(request.get("tool_choice")) started = time.monotonic() response = await self._coerce_response(await self._send(request)) + # toolChoice constrains model output, so a forced requirement is + # satisfied as soon as this turn emits a tool call. Record it before + # any approval/HITL/client pause can persist the run. + self._commit_tool_choice_dispatch(response_has_tool_calls(response)) await self._emit_post_model_call(response, started, turn_type, turn_number) return response + # -- forced tool-choice policy (DEV-785) --------------------------------- + + def _apply_resolved_forced_tool_choice_policy(self, request: Dict[str, Any]) -> Dict[str, Any]: + """Record the freshly-resolved caller choice before deriving its wire + value. An unforced turn clears the consumed key, re-arming the same + forced value if it appears again later. A different forced key + re-arms immediately.""" + self._configured_tool_choice = request.get("tool_choice") + self._configured_forced_tool_choice_key = _forced_tool_choice_key(self._configured_tool_choice) + if self._configured_forced_tool_choice_key is None: + self._prepared_tool_choice_commit = ( + None if self._consumed_forced_tool_choice_key is None else ("clear", None) + ) + elif self._configured_forced_tool_choice_key == self._consumed_forced_tool_choice_key: + self._prepared_tool_choice_commit = None + else: + self._prepared_tool_choice_commit = ("consume", self._configured_forced_tool_choice_key) + return self._apply_forced_tool_choice_policy(request) + + def _apply_forced_tool_choice_policy(self, request: Dict[str, Any]) -> Dict[str, Any]: + """Derive the caller-configured tool choice for the active logical run. + A forced choice that already produced a tool call is relaxed so the + model can answer in text. Engine-owned one-turn overrides (`none` for + finalization) are applied after this helper and therefore still win.""" + if ( + self._configured_forced_tool_choice_key is not None + and self._configured_forced_tool_choice_key == self._consumed_forced_tool_choice_key + ): + tool_choice = _relax_forced_tool_choice(self._configured_tool_choice) + else: + tool_choice = self._configured_tool_choice + self._prepared_effective_tool_choice = tool_choice + if tool_choice is request.get("tool_choice"): + return request + updated = dict(request) + if tool_choice is None: + updated.pop("tool_choice", None) + else: + updated["tool_choice"] = tool_choice + return updated + + def _begin_tool_choice_dispatch(self, actual_tool_choice: Any) -> None: + """Bind the prepared caller-policy transition to the actual wire + request. Engine overrides produce a different key and cannot commit.""" + self._dispatched_caller_tool_choice = _tool_choice_identity(actual_tool_choice) == _tool_choice_identity( + self._prepared_effective_tool_choice + ) + self._dispatched_tool_choice_commit = ( + self._prepared_tool_choice_commit if self._dispatched_caller_tool_choice else None + ) + + def _commit_tool_choice_dispatch(self, has_tool_calls: bool) -> None: + """Commit only after the prepared caller choice reached the wire and a + response materialized; a forced choice is consumed only when that + response called a tool.""" + dispatched = self._dispatched_caller_tool_choice + commit = self._dispatched_tool_choice_commit + self._dispatched_caller_tool_choice = False + self._dispatched_tool_choice_commit = None + if not dispatched: + return + self._prepared_tool_choice_commit = None + if commit is not None and commit[0] == "clear": + self._consumed_forced_tool_choice_key = None + elif commit is not None and commit[0] == "consume" and has_tool_calls: + self._consumed_forced_tool_choice_key = commit[1] + + def _reset_forced_tool_choice_policy(self) -> None: + """Clear all runtime policy state for a terminal run exit: a later + conversational turn is a new logical run and may force again.""" + self._consumed_forced_tool_choice_key = None + self._configured_forced_tool_choice_key = None + self._configured_tool_choice = None + self._prepared_effective_tool_choice = None + self._prepared_tool_choice_commit = None + self._dispatched_tool_choice_commit = None + self._dispatched_caller_tool_choice = False + + async def _mark_state_complete(self) -> None: + self._reset_forced_tool_choice_policy() + await self._save_state(status="complete") + async def _append_event(self, event: Any) -> None: async with self._condition: self._events.append(event) @@ -222,14 +498,50 @@ async def _load_state(self) -> None: loaded = await maybe_await(accessor.load()) self._resuming_from_client_tools = bool(loaded is not None and loaded.status == "awaiting_client_tools") self._state = loaded or create_initial_state() - if not self._resuming_from_client_tools: - self._state = update_state(self._state, {"status": "in_progress"}) - await maybe_await(accessor.save(self._state)) + if loaded is not None: + self._consumed_forced_tool_choice_key = getattr(loaded, "consumed_forced_tool_choice_key", None) + self._restore_doom_loop_state(getattr(loaded, "doom_loop", None)) + + def _restore_doom_loop_state(self, persisted: Any) -> None: + """Rehydrate doom-loop state so a resumed run continues where it left + off: streak counters always; the queued steer guidance; and the stop + verdict under the condemnation rule -- a doom-stopped conversation + STAYS stopped across decision-only resumes (approve/reject), while a + fresh conversational turn clears it (operator input is new + information). Streaks are kept either way.""" + monitor = self._doom_loop_monitor + if monitor is None or persisted is None: + return + monitor.restore(persisted) + if not isinstance(persisted, Mapping): + return + pending_steer = persisted.get("pending_steer", persisted.get("pendingSteer")) + if isinstance(pending_steer, list): + for message in pending_steer: + if isinstance(message, str): + self._queue_doom_loop_steer(message) + stop_verdict = persisted.get("stop_verdict", persisted.get("stopVerdict")) + if isinstance(stop_verdict, Mapping) and stop_verdict and (self._approved_ids or self._rejected_ids): + self._doom_loop_stop = cast(DoomLoopVerdict, dict(stop_verdict)) async def _save_state(self, **updates: Any) -> None: accessor = self.options.get("state") if accessor is None or self._state is None: return + # Persist only the consumed semantic tool-choice key, never the + # callback or a copied effective tool_choice. + updates["consumed_forced_tool_choice_key"] = self._consumed_forced_tool_choice_key + # Piggyback the doom-loop detector snapshot on every persist so loop + # memory survives serialize -> resume. Engine-owned fields ride along: + # `stop_verdict` keeps a condemned run condemned across decision-only + # resumes; `pending_steer` carries guidance that had no flush point. + if self._doom_loop_monitor is not None: + doom_loop_state: Dict[str, Any] = dict(self._doom_loop_monitor.get_state()) + if self._doom_loop_stop is not None: + doom_loop_state["stop_verdict"] = dict(self._doom_loop_stop) + if self._pending_doom_loop_steer: + doom_loop_state["pending_steer"] = list(self._pending_doom_loop_steer) + updates["doom_loop"] = doom_loop_state self._state = update_state(self._state, updates) await maybe_await(accessor.save(self._state)) @@ -278,12 +590,26 @@ async def _persist_client_tools_pause(self, response: Any, unresolved_calls: Seq def _validate_final_response(self, response: Any, allow_empty_output: bool = False) -> None: response_id = get_field(response, "id") output = get_field(response, "output") + # Message prefixes ("Invalid final response: missing required fields", + # "Invalid final response: empty or invalid output") are load-bearing -- + # callers and tests match on them. Detail is appended after an em dash. if not response_id or output is None: - raise ValueError("Invalid final response: missing required fields") + missing = [ + name for name, present in (("id", bool(response_id)), ("output", output is not None)) if not present + ] + raise ValueError(f"Invalid final response: missing required fields: {', '.join(missing)}") if not isinstance(output, list) or len(output) == 0: if allow_empty_output: return - raise ValueError("Invalid final response: empty or invalid output") + if isinstance(output, list): + detail = ( + f'output array is empty (length 0) for response "{response_id}". The model returned no ' + "output items. This can happen when the provider returns an empty final turn; see the " + "strict_final_response/allow_final_response options" + ) + else: + detail = f'output is not an array (got {_describe_non_record(output)}) for response "{response_id}"' + raise ValueError(f"Invalid final response: empty or invalid output \u2014 {detail}") # -- hooks ----------------------------------------------------------- @@ -315,25 +641,32 @@ async def _emit_session_end(self, reason: str) -> None: self._session_end_emitted = True payload: Dict[str, Any] = {"reason": reason} if self._session_usage["model_calls"] > 0: - total_usage = { - "model_calls": self._session_usage["model_calls"], - "input_tokens": self._session_usage["input_tokens"], - "output_tokens": self._session_usage["output_tokens"], - "total_tokens": self._session_usage["total_tokens"], - "cached_tokens": self._session_usage["cached_tokens"], - "reasoning_tokens": self._session_usage["reasoning_tokens"], - } - if self._session_usage["has_cost"]: - total_usage["cost"] = self._session_usage["cost"] - payload["total_usage"] = total_usage + payload["total_usage"] = self._snapshot_session_usage() try: await self._hooks.emit("SessionEnd", payload, session_id=self._session_id) except Exception as error: # noqa: BLE001 - teardown must never mask the real error warnings.warn(f"[SessionEnd] error during session teardown: {error}", stacklevel=2) + def _snapshot_session_usage(self) -> Dict[str, Any]: + """Single source of truth for `SessionEnd.total_usage` and the public + `get_usage()` accessor, so the two can never drift. `cost` is present + only when at least one response reported it.""" + totals: Dict[str, Any] = { + "model_calls": self._session_usage["model_calls"], + "input_tokens": self._session_usage["input_tokens"], + "output_tokens": self._session_usage["output_tokens"], + "total_tokens": self._session_usage["total_tokens"], + "cached_tokens": self._session_usage["cached_tokens"], + "reasoning_tokens": self._session_usage["reasoning_tokens"], + } + if self._session_usage["has_cost"]: + totals["cost"] = self._session_usage["cost"] + return totals + async def _emit_post_model_call(self, response: Any, started_at: float, turn_type: str, turn_number: int) -> None: - if not self._hooks: - return + # Accumulation runs unconditionally -- before the hooks short-circuit -- + # because `get_usage()` surfaces these totals to callers who configured + # no hooks at all. Only the hook emit is gated on a HooksManager. usage = _extract_model_call_usage(get_field(response, "usage")) self._session_usage["model_calls"] += 1 if usage: @@ -345,6 +678,8 @@ async def _emit_post_model_call(self, response: Any, started_at: float, turn_typ if usage.get("cost") is not None: self._session_usage["cost"] += usage["cost"] self._session_usage["has_cost"] = True + if not self._hooks: + return payload = { "session_id": self._session_id, "response_id": get_field(response, "id", ""), @@ -392,6 +727,117 @@ async def _emit_permission_request(self, call: ParsedToolCall, tools: Sequence[T last = emitted.results[-1] return last.get("decision", "ask_user"), last.get("reason") + # -- two-phase approval preparation ----------------------------------------- + + async def _prepare_tool_call_for_approval(self, call: ParsedToolCall) -> ParsedToolCall: + """Run PreToolUse ahead of execution and park its outcome in + `_prepared_tool_calls` (upstream `prepareToolCallForApproval`). + Returns the effective call (the mutated input when a handler piped + one). Raw-string (unparseable) arguments never reach the hook.""" + if not self._hooks or isinstance(call.arguments, str): + return call + pre = await self._hooks.emit( + "PreToolUse", + {"tool_name": call.name, "tool_input": call.arguments if isinstance(call.arguments, dict) else {}}, + tool_name=call.name, + session_id=self._session_id, + ) + if pre.blocked: + block = next((r.get("block") for r in pre.results if r.get("block")), None) + reason = block if isinstance(block, str) else "Blocked by PreToolUse hook" + self._prepared_tool_calls[call.id] = {"type": "blocked", "reason": reason} + return call + effective = dataclasses.replace(call, arguments=pre.final_payload.get("tool_input")) if pre.mutated else call + self._prepared_tool_calls[call.id] = {"type": "ready", "call": effective, "mutated": bool(pre.mutated)} + return effective + + async def _mutated_input_requires_approval( + self, call: ParsedToolCall, context: Mapping[str, Any], tools: Sequence[Tool] + ) -> bool: + """Re-check only approval sources whose answer can depend on input: + the call-level `require_approval` option, or a callable tool-level + `require_approval` (upstream `mutatedInputRequiresApproval`).""" + if not tools: + return False + call_level = self.options.get("require_approval") + if call_level is not None: + return await tool_requires_approval(call, tools, context, call_level) + tool = self._find_tool(call.name, tools) + if tool is None or not callable(get_tool_function(tool).get("require_approval")): + return False + return await tool_requires_approval(call, tools, context) + + async def _emit_prepared_failure(self, call: ParsedToolCall, reason: str) -> None: + """PostToolUseFailure for a call whose lifecycle entered PreToolUse but + never executed (mutated-input denial/rejection, invalid mutation).""" + if not self._hooks: + return + await self._hooks.emit( + "PostToolUseFailure", + { + "tool_name": call.name, + "tool_input": call.arguments if isinstance(call.arguments, dict) else {}, + "error": reason, + }, + tool_name=call.name, + session_id=self._session_id, + ) + + async def _block_prepared_tool_call(self, call: ParsedToolCall, reason: str) -> None: + self._prepared_tool_calls[call.id] = {"type": "blocked", "reason": reason} + await self._emit_prepared_failure(call, reason) + + async def _validate_prepared_mutation(self, effective: ParsedToolCall, tools: Sequence[Tool]) -> str: + """Validate hook-mutated input against the tool's input schema before + it can be re-gated or executed. Returns "ready" or "blocked".""" + tool = self._find_tool(effective.name, tools) + if tool is None: + return "ready" + try: + parsed = validate_schema(get_tool_function(tool).get("input_schema"), effective.arguments) + except Exception: # noqa: BLE001 - any validation failure blocks the call + parsed = None + valid = False + else: + valid = True + if not valid or not isinstance(parsed, (Mapping, BaseModel)): + await self._block_prepared_tool_call(effective, f'PreToolUse produced invalid input for "{effective.name}"') + return "blocked" + return "ready" + + async def _prepare_after_initial_approval( + self, + call: ParsedToolCall, + context: Mapping[str, Any], + tools: Sequence[Tool], + gate_key: Tuple[Any, Any], + ) -> str: + """Second approval phase for a call that passed (or skipped) the + initial gate (upstream `prepareAfterInitialApproval`): run PreToolUse, + validate a mutated input, and re-gate it once when an input-dependent + approval source now requires approval. Returns "ready", "blocked", or + "pending" (the mutated call needs a human decision).""" + effective = await self._prepare_tool_call_for_approval(call) + prepared = self._prepared_tool_calls.get(call.id) + if prepared is None or prepared["type"] != "ready" or not prepared["mutated"]: + return "blocked" if prepared is not None and prepared["type"] == "blocked" else "ready" + + if await self._validate_prepared_mutation(effective, tools) == "blocked": + return "blocked" + if gate_key in self._completed_mutated_approval_gates: + return "ready" + self._completed_mutated_approval_gates.add(gate_key) + if not await self._mutated_input_requires_approval(effective, context, tools): + return "ready" + + decision, reason = await self._emit_permission_request(effective, tools) + if decision == "allow": + return "ready" + if decision == "deny": + await self._block_prepared_tool_call(effective, reason or "Denied by PermissionRequest hook") + return "blocked" + return "pending" + async def _maybe_run_user_prompt_submit(self, input_value: Any) -> Any: if not self._hooks or input_value is None: return input_value @@ -470,16 +916,25 @@ def _find_tool(self, name: str, tools: Sequence[Tool]) -> Optional[Tool]: return next((t for t in tools if is_client_tool(t) and get_tool_function(t).get("name") == name), None) async def _run_tool_with_hooks( - self, tool: Tool, call: ParsedToolCall, context: Mapping[str, Any] + self, + tool: Tool, + call: ParsedToolCall, + context: Mapping[str, Any], + run_pre_tool_use: bool = True, ) -> Dict[str, Any]: """Execute a single tool and emit the PreToolUse/PostToolUse/ PostToolUseFailure lifecycle hooks around it. + PreToolUse normally already ran during approval preparation + (`_prepare_tool_call_for_approval`); its parked outcome is consumed + here so the hook never runs twice for one call. It is emitted here + only when no prepared outcome exists and `run_pre_tool_use` is True. + Returns a tagged outcome: - `{"type": "parse_error", "call", "error_message"}` -- the model produced invalid JSON for this call's arguments. No hooks fire. - `{"type": "hook_blocked", "call", "reason"}` -- PreToolUse blocked - the call. + the call (or its prepared outcome was blocked). - `{"type": "execution", "call", "result"}` -- the tool ran (or paused, if `result` is `None` for a HITL tool). `call` reflects any `mutated_input` piped by PreToolUse. @@ -492,8 +947,14 @@ async def _run_tool_with_hooks( ) return {"type": "parse_error", "call": call, "error_message": error_message} - effective_call = call - if self._hooks: + prepared = self._prepared_tool_calls.pop(call.id, None) + if prepared is not None and prepared["type"] == "blocked": + return {"type": "hook_blocked", "call": call, "reason": prepared["reason"]} + + effective_call: ParsedToolCall = prepared["call"] if prepared is not None else call + # Emit PreToolUse here only when the approval gate has not already + # done so (e.g. an approved call resumed from legacy persisted state). + if self._hooks and prepared is None and run_pre_tool_use: original_input = call.arguments if isinstance(call.arguments, dict) else {} pre = await self._hooks.emit( "PreToolUse", @@ -544,11 +1005,14 @@ async def _run_tool_with_hooks( return {"type": "execution", "call": effective_call, "result": result} async def _record_preliminary(self, call_id: str, value: Any) -> None: + name = self._call_names.get(call_id, "") await self._append_event( { "type": "tool.preliminary_result", "toolCallId": call_id, "tool_call_id": call_id, + "toolName": name, + "tool_name": name, "result": value, "timestamp": _now_ms(), } @@ -579,6 +1043,836 @@ def _rejected_output(self, call_id: str, reason: str) -> Dict[str, Any]: "output": json_dumps({"error": reason}), } + # -- doom-loop detection ------------------------------------------------- + + async def _begin_doom_loop_round(self, batch: Sequence[ParsedToolCall], tools: Sequence[Tool]) -> None: + """Advance the doom-loop round counter and declare the round's calls + (upstream `beginDoomLoopRound`). A round's identity for one tool is + the *set* of fingerprints it was called with, so the set must be + complete before any of its calls is scored. + + INVARIANT -- every declared member must eventually be recorded. The + filters below therefore mirror every path that skips the per-call + checkpoint: unknown/manual tools (`is_auto_resolvable_tool`), calls a + PermissionRequest hook denied, and `loop_key`-exempt calls. A + declared-but-never-recorded phantom member silently resets the + tool's round streak. + """ + monitor = self._doom_loop_monitor + if monitor is None: + return + self._doom_loop_round += 1 + self._doom_loop_round_decisions.clear() + self._doom_loop_round_key_material.clear() + + declared: List[Any] = [] + for call in batch: + tool = self._find_tool(call.name, tools) + # Never checked => never recorded => must not be declared (and + # its `loop_key` must not run). Precedes the malformed branch: a + # raw-string call to an unknown or manual tool is never recorded. + if tool is None or not is_auto_resolvable_tool(tool): + continue + if call.id in self._hook_denied_calls: + continue + if isinstance(call.arguments, str): + declared.append({"tool_name": str(call.name), "key_material": call.arguments}) + continue + # `resolve_loop_key_material` catches a raising `loop_key`, but the + # field-list form reads the arguments, which can raise on its own. + # Skip just that call; the per-call checkpoint retries it. + try: + resolution = resolve_loop_key_material( + get_tool_function(tool).get("loop_key"), + call.arguments if call.arguments is not None else {}, + ) + except Exception as error: # noqa: BLE001 - detection must never fail a run + warnings.warn( + f'[DoomLoop] could not resolve loop identity for "{call.name}" while declaring the round; ' + f"excluding it from the round set: {error}", + stacklevel=2, + ) + continue + # Cache so the checkpoint does not invoke `loop_key` again. A + # duplicate (model-emitted) call id poisons the entry so neither + # call aliases the other's identity. + if call.id in self._doom_loop_round_key_material: + self._doom_loop_round_key_material[call.id] = _DUPLICATE_CALL_ID + else: + self._doom_loop_round_key_material[call.id] = resolution + if resolution["kind"] == "exempt": + continue + declared.append({"tool_name": str(call.name), "key_material": resolution["key_material"]}) + await monitor.declare_round(self._doom_loop_round, declared) + + async def _evaluate_doom_loop_call( + self, + *, + tool_name: str, + key_material: Any, + allow_block: bool, + detector: Literal["tool-fingerprint", "server-tool-fingerprint"], + fallback_key_material: Any = _NO_FALLBACK, + call: Optional[ParsedToolCall] = None, + ) -> Dict[str, Any]: + """Evaluate one call against the monitor (upstream + `enqueueDoomLoopEvaluation`). Upstream serializes evaluations on a + promise chain because its round executes in parallel; this port runs + a round's calls sequentially in emission order, so plain awaiting + gives the same deterministic recording order. + + Unhashable key material degrades to `fallback_key_material`; if there + is none (or it is also unhashable) detection is skipped for the call. + Returns `{"action": ..., "message"?: ...}` (`"proceed"` when no rung + fired). Shared side effects (steer queue, escalation, stop arm) are + applied here. + """ + monitor = self._doom_loop_monitor + if monitor is None: + return {"action": "proceed"} + try: + record = await monitor.record_tool_call( + tool_name, key_material, self._doom_loop_round, allow_block=allow_block, detector=detector + ) + except Exception as error: # noqa: BLE001 - detection must never fail a call + has_fallback = fallback_key_material is not _NO_FALLBACK + warnings.warn( + f'[DoomLoop] could not fingerprint call to "{tool_name}"; ' + f"{'falling back to full arguments' if has_fallback else 'skipping detection for this call'}: " + f"{error}", + stacklevel=2, + ) + if not has_fallback: + return {"action": "proceed"} + try: + record = await monitor.record_tool_call( + tool_name, fallback_key_material, self._doom_loop_round, allow_block=allow_block, detector=detector + ) + except Exception as fallback_error: # noqa: BLE001 + warnings.warn( + f'[DoomLoop] fallback identity for "{tool_name}" also unhashable; ' + f"skipping detection for this call: {fallback_error}", + stacklevel=2, + ) + return {"action": "proceed"} + + # Same (tool, fingerprint) already decided this round: reuse the + # decision -- no second streak increment, no second hook emission. + decision_key = f"{tool_name}\n{record['fingerprint']}" + if record["duplicate_in_round"]: + return self._doom_loop_round_decisions.get(decision_key, {"action": "proceed"}) + verdict = record.get("verdict") + if verdict is None: + decision: Dict[str, Any] = {"action": "proceed"} + self._doom_loop_round_decisions[decision_key] = decision + return decision + action = await self._apply_doom_loop_verdict(verdict, call) + decision = {"action": action, "message": verdict["message"]} + self._doom_loop_round_decisions[decision_key] = decision + self._apply_doom_loop_side_effects(action, verdict) + return decision + + async def _check_doom_loop_before_execution(self, tool: Tool, call: ParsedToolCall) -> Tuple[bool, str]: + """Doom-loop checkpoint for one parsed call, before PreToolUse and + execution (upstream `checkDoomLoopBeforeExecution`). Returns + `(blocked, reason)`; `stop` also blocks the condemned call.""" + if self._doom_loop_monitor is None: + return False, "" + call_arguments = call.arguments if call.arguments is not None else {} + cached = self._doom_loop_round_key_material.get(call.id) + resolution: LoopKeyResolution + if cached is not None and cached is not _DUPLICATE_CALL_ID: + resolution = cast(LoopKeyResolution, cached) + else: + try: + resolution = resolve_loop_key_material(get_tool_function(tool).get("loop_key"), call_arguments) + except Exception as error: # noqa: BLE001 - detection must never fail a run + warnings.warn( + f'[DoomLoop] could not resolve loop identity for "{call.name}"; ' + f"skipping detection for this call: {error}", + stacklevel=2, + ) + return False, "" + if resolution["kind"] == "exempt": + return False, "" + key_material = resolution["key_material"] + if resolution["kind"] == "fallback": + warnings.warn(f'[DoomLoop] tool "{call.name}": {resolution["warning"]}', stacklevel=2) + decision = await self._evaluate_doom_loop_call( + tool_name=str(call.name), + key_material=key_material, + fallback_key_material=_NO_FALLBACK if key_material is call_arguments else call_arguments, + allow_block=True, + detector="tool-fingerprint", + call=call, + ) + if decision["action"] in ("block", "stop"): + return True, decision.get("message") or ( + f'Doom loop suspected: tool "{call.name}" is repeating identical calls. ' + "Repeating the call will not change the result. Take a different approach." + ) + return False, "" + + async def _apply_doom_loop_verdict( + self, verdict: DoomLoopVerdict, call: Optional[ParsedToolCall] = None + ) -> DoomLoopAction: + """Emit the DoomLoopDetected hook and resolve the final action. The + LAST `override_action` in the chain wins. Non-blockable verdicts + (text, server tools) downgrade 'block' to 'observe'; 'escalate' + without a mechanism or budget downgrades to 'observe'.""" + action: DoomLoopAction = verdict["action"] + if self._hooks: + payload: Dict[str, Any] = { + "detector": verdict["detector"], + "action": verdict["action"], + "streak": verdict["streak"], + "fingerprint": verdict["fingerprint"], + "message": verdict["message"], + } + if "tool_name" in verdict: + payload["tool_name"] = verdict["tool_name"] + if call is not None and isinstance(call.arguments, dict): + payload["tool_input"] = call.arguments + emitted = await self._hooks.emit( + "DoomLoopDetected", + payload, + tool_name=verdict.get("tool_name"), + session_id=self._session_id, + ) + for result in emitted.results: + override = result.get("override_action") if isinstance(result, Mapping) else None + if override: + action = override + if verdict["detector"] != "tool-fingerprint" and action == "block": + action = "observe" + if action == "escalate" and not ( + self._doom_loop_monitor is not None and self._doom_loop_monitor.can_escalate() + ): + action = "observe" + return action + + def _apply_doom_loop_side_effects(self, action: str, verdict: DoomLoopVerdict) -> None: + """Side effects shared by every checkpoint: queue steer guidance, arm + an escalation for the next request, or arm the stop verdict. 'block' + has none -- the calling checkpoint synthesizes the blocked output.""" + if action == "steer": + self._queue_doom_loop_steer(verdict["message"]) + elif action == "escalate": + # Latch the FIRST escalation verdict for the next request: a second + # verdict in the same window must not double-spend the budget. + if self._pending_doom_loop_escalation is None: + self._pending_doom_loop_escalation = verdict + self._queue_doom_loop_steer( + f"{verdict['message']} An escalated turn follows: use the additional guidance to change course." + ) + elif action == "stop": + self._doom_loop_stop = verdict + + def _apply_doom_loop_escalation(self, request: Mapping[str, Any]) -> Mapping[str, Any]: + """Consume a pending escalation into ONE-TURN request overrides + (upstream `takeDoomLoopEscalationOverrides`) and return the request to + dispatch. The caller's request is never mutated, so the following turn + reverts automatically. Burns one unit of the budget at APPLICATION + time. + + - `model`: replace the model for this dispatch (dropping any `models` + fallback list so it cannot shadow the override). + - `advisor`: append an `openrouter:advisor` server tool and pin + `tool_choice` to it so the model must consult it this turn. + """ + verdict = self._pending_doom_loop_escalation + config = self._doom_loop_escalation + monitor = self._doom_loop_monitor + self._pending_doom_loop_escalation = None + if verdict is None or config is None or monitor is None or not monitor.can_escalate(): + return request + monitor.consume_escalation() + escalated: Dict[str, Any] = dict(request) + if config.get("model") is not None: + escalated["model"] = config["model"] + escalated.pop("models", None) + advisor = config.get("advisor") + if advisor is not None and advisor is not False: + # Upstream spells the parameter `forwardTranscript`; this port's + # wire is the generated Python SDK's snake_case + # (`AdvisorServerToolConfig.forward_transcript`). + advisor_parameters: Dict[str, Any] = { + "forward_transcript": True, + "instructions": ( + "You are an escalation advisor. The executing model appears stuck in a loop: " + f"{verdict['message']} Diagnose why its approach is failing and give concrete, " + "specific instructions for a DIFFERENT approach. Do not restate the problem." + ), + **(advisor if isinstance(advisor, Mapping) else {}), + } + existing_tools = request.get("tools") + escalated["tools"] = [ + *(existing_tools if isinstance(existing_tools, list) else []), + {"type": "openrouter:advisor", "parameters": advisor_parameters}, + ] + escalated["tool_choice"] = { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "openrouter:advisor"}], + } + return escalated + + async def _check_doom_loop_for_response(self, response: Any) -> None: + """Step-level checkpoint on a fresh model response (upstream + `checkDoomLoopForResponse`): the text detectors, then server-tool + fingerprints. Server tools never pass through `_run_tool_with_hooks`, + so their repetition is detected here, post-execution (no block).""" + monitor = self._doom_loop_monitor + if monitor is None: + return + verdict: Optional[DoomLoopVerdict] + try: + verdict = await monitor.record_assistant_text(extract_text_from_response(response)) + except Exception as error: # noqa: BLE001 - detection must never fail a run + warnings.warn(f"[DoomLoop] text checkpoint failed; skipping for this response: {error}", stacklevel=2) + verdict = None + if verdict is not None: + action = await self._apply_doom_loop_verdict(verdict) + self._apply_doom_loop_side_effects(action, verdict) + + for item in self._response_output_items(response): + item_type = get_field(item, "type") + if item_type is None or item_type in ("message", "reasoning", "function_call"): + continue + identity = _extract_server_tool_identity(item) + if identity is None: + continue + await self._evaluate_doom_loop_call( + tool_name=f"server:{item_type}", + key_material=identity, + allow_block=False, + detector="server-tool-fingerprint", + ) + + def _queue_doom_loop_steer(self, message: str) -> None: + """Queue steer guidance, deduplicating identical messages.""" + if message not in self._pending_doom_loop_steer: + self._pending_doom_loop_steer.append(message) + + async def _flush_doom_loop_steer(self, current_request: Dict[str, Any]) -> None: + """Inject queued steer guidance as one user message before the next + follow-up request. Only called at a safe point (a fully-paired round): + a user message between a dangling function_call and its output would + be invalid history, so pauses persist the queue instead.""" + if not self._pending_doom_loop_steer: + return + prompt = "\n".join(self._pending_doom_loop_steer) + self._pending_doom_loop_steer = [] + await self._inject_append_prompt_message(prompt, current_request) + + async def _seal_doom_loop_stop(self, response: Any, resolved_call_ids: Optional[set[str]] = None) -> None: + """Halt for an armed stop verdict keeping persisted history + well-formed: every function_call in `response` without an output yet + gets a synthesized halt-error output, so a stateful resume never sends + a dangling function_call (providers reject those with a 400).""" + verdict = self._doom_loop_stop + unresolved = [ + call + for call in extract_tool_calls_from_response(response) + if resolved_call_ids is None or call.id not in resolved_call_ids + ] + if not unresolved: + return + halt_message = ( + f"Run halted by doom-loop detection: {verdict['message']}" + if verdict is not None + else "Run halted by doom-loop detection." + ) + await self._save_tool_outputs_to_state([self._rejected_output(call.id, halt_message) for call in unresolved]) + + # -- approval gate ------------------------------------------------------- + + def _is_decision_resume(self) -> bool: + """Resuming an approval/HITL pause with explicit approve/reject + decisions (`process_approval_decisions` upstream).""" + state = self._state + return bool( + state is not None + and state.status in ("awaiting_approval", "awaiting_hitl") + and state.pending_tool_calls + and (self._approved_ids or self._rejected_ids) + ) + + def _stays_paused_without_decisions(self, resolved_input: Any) -> bool: + """Paused for a human decision but none was supplied: stay paused. + + Falling through would flip the status to in_progress and send a fresh + model request, advancing the conversation past calls that still owe a + decision. An `awaiting_hitl` conversation still resumes the documented + way -- a `function_call_output` for a paused call in `input`. + """ + state = self._state + if state is None or state.status not in ("awaiting_approval", "awaiting_hitl"): + return False + if self._approved_ids or self._rejected_ids: + return False + pending_ids = {call.id for call in state.pending_tool_calls or []} + if not pending_ids: + return False + if state.status != "awaiting_hitl": + return True + if not isinstance(resolved_input, list): + return True + return not any( + get_field(item, "type") == "function_call_output" + and get_field(item, "callId", get_field(item, "call_id")) in pending_ids + for item in resolved_input + ) + + def _paused_error(self) -> Optional[Exception]: + """Describe the pause when the conversation is waiting on the caller + (`awaiting_approval` / `awaiting_hitl`), or None when it is not.""" + state = self._state + status = state.status if state is not None else None + if status not in ("awaiting_approval", "awaiting_hitl"): + return None + pending_ids = [call.id for call in (state.pending_tool_calls or [])] + shown = ", ".join(pending_ids[:_MAX_PENDING_IDS_IN_ERROR]) + hidden = len(pending_ids) - _MAX_PENDING_IDS_IN_ERROR + id_list = f"{shown}, ... ({hidden} more)" if hidden > 0 else shown + if status == "awaiting_approval": + what = "awaiting approval" + how = "Resume with approve_tool_calls and/or reject_tool_calls for the pending calls." + else: + what = "awaiting HITL input" + how = ( + "Resume by passing a function_call_output for a pending call ID in input, " + "or with approve_tool_calls / reject_tool_calls (approving re-runs on_tool_called)." + ) + raw_input = self.options["request"].get("input") + input_supplied = callable(raw_input) or bool(raw_input) + return RuntimeError( + f"Conversation is {what} (status '{status}'); no response is available. " + f"Pending tool call IDs: {id_list}. {how} " + "Use requires_approval() or get_pending_tool_calls() to inspect a paused conversation." + + ( + " The input supplied with this call was not sent to the model and was not saved to state." + if input_supplied + else "" + ) + ) + + def _response_unavailable_error(self, fallback: str = "Response not available") -> Exception: + return self._paused_error() or RuntimeError(fallback) + + async def _execute_auto_approve_tools( + self, calls: Sequence[ParsedToolCall], tools: Sequence[Tool], turn_context: Mapping[str, Any] + ) -> List[UnsentToolResult]: + """Execute auto-approved calls while pausing for approval and keep + their results as unsent, so the resume path replays them without + re-executing.""" + results: List[UnsentToolResult] = [] + for call in calls: + tool = self._find_tool(call.name, tools) + if not tool or not is_auto_resolvable_tool(tool): + continue + try: + outcome = await self._run_tool_with_hooks(tool, call, turn_context) + except Exception as error: # noqa: BLE001 - mirrors upstream Promise.allSettled + results.append(create_rejected_result(call.id, call.name, str(error))) + continue + if outcome["type"] == "parse_error": + await self._append_tool_result_event(call, tool, {"error": outcome["error_message"]}) + results.append(create_rejected_result(call.id, call.name, outcome["error_message"])) + elif outcome["type"] == "hook_blocked": + results.append(create_rejected_result(call.id, call.name, outcome["reason"])) + elif outcome["result"] is None: + # HITL tool paused -- no unsent result for this call. + continue + elif outcome["result"].get("error") is not None: + results.append(create_rejected_result(call.id, call.name, str(outcome["result"]["error"]))) + else: + results.append(create_unsent_result(call.id, call.name, outcome["result"].get("result"))) + return results + + async def _handle_approval_check( + self, + calls: Sequence[ParsedToolCall], + current_round: int, + response: Any, + tools: Sequence[Tool], + skip_auto_execution: bool = False, + ) -> bool: + """Gate a response's tool calls on approval. Returns True when the run + must pause (`awaiting_approval`). + + Each call occurrence in a response is gated at most once per run, so + the pre-loop gate, the in-loop gate, and the `allow_final_response` + gate never re-emit `PermissionRequest` or re-run predicates for an + already-resolved call. Nothing is executed when no call needs a + gate -- the normal tool round runs every call exactly once. When + pausing, auto-approved siblings run now (or, with + `skip_auto_execution`, are recorded as skipped) and are persisted as + unsent results for the resume. + """ + if not tools: + return False + unseen, blocked = await self._classify_initial_approval_calls(calls, response, tools) + if not unseen and not blocked: + return False + + turn_context = {"number_of_turns": current_round} + partition = await partition_tool_calls(unseen, tools, turn_context, self.options.get("require_approval")) + needs_approval = list(partition["requires_approval"]) + auto_execute = list(partition["auto_execute"]) + + denied, still_pending, initial_survivors = await self._resolve_approval_phases( + needs_approval, auto_execute, calls, response, tools, turn_context + ) + + if not still_pending: + for call, reason in denied: + self._hook_denied_calls[call.id] = reason + return False + + if self.options.get("state") is None: + names = ", ".join(call.name for call in still_pending) + raise ValueError( + f"Tool(s) require approval but no state accessor is configured: {names}. " + 'Provide a StateAccessor via the "state" parameter to enable approval workflows.' + ) + + pending_ids = {call.id for call in still_pending} + # Doom-blocked calls ride along: `_execute_auto_approve_tools` turns + # them into rejected unsent results without executing them. Survivors + # consume their prepared PreToolUse outcome there. + executable_now = [*[c for c in initial_survivors if c.id not in pending_ids], *blocked] + if skip_auto_execution: + unsent = [ + create_rejected_result(call.id, call.name, "Tool execution skipped: stop condition met") + for call in executable_now + ] + else: + unsent = await self._execute_auto_approve_tools(executable_now, tools, turn_context) + combined = [*unsent, *[create_rejected_result(call.id, call.name, reason) for call, reason in denied]] + + # A pending call whose PreToolUse already ran (it paused on the + # mutated-input gate) persists its effective arguments plus the + # marker, so the resume does not run the hook again. + pending_to_save = [ + dataclasses.replace(call, pre_tool_use_applied=True) if call.id in self._prepared_tool_calls else call + for call in still_pending + ] + save_kwargs: Dict[str, Any] = {"pending_tool_calls": pending_to_save, "status": "awaiting_approval"} + if combined: + save_kwargs["unsent_tool_results"] = combined + await self._save_state(**save_kwargs) + self._prepared_tool_calls.clear() + self._hook_denied_calls.clear() + self._final_response = response + return True + + async def _resolve_approval_phases( + self, + needs_approval: Sequence[ParsedToolCall], + auto_execute: Sequence[ParsedToolCall], + calls: Sequence[ParsedToolCall], + response: Any, + tools: Sequence[Tool], + turn_context: Mapping[str, Any], + ) -> Tuple[List[Tuple[ParsedToolCall, str]], List[ParsedToolCall], List[ParsedToolCall]]: + """Run both approval phases (upstream `resolveApprovalPhases`). + + Phase 1 gates the model-issued arguments: PermissionRequest for every + call needing approval. Phase 2 runs PreToolUse for every + auto-resolvable survivor (auto-approved or hook-allowed) so its + effective arguments are known before execution; a mutated input is + validated and re-gated by input-dependent approval sources. Returns + `(denied, still_pending, initial_survivors)`. + """ + denied: List[Tuple[ParsedToolCall, str]] = [] + still_pending: List[ParsedToolCall] = [] + if self._hooks: + for call in needs_approval: + decision, reason = await self._emit_permission_request(call, tools) + if decision == "deny": + denied.append((call, reason or "Denied by PermissionRequest hook")) + elif decision != "allow": + still_pending.append(call) + else: + still_pending = list(needs_approval) + + pending_ids = {call.id for call in still_pending} + denied_ids = {call.id for call, _ in denied} + initial_survivors = [ + *auto_execute, + *[c for c in needs_approval if c.id not in pending_ids and c.id not in denied_ids], + ] + # Occurrence identity of each call within its response, matching the + # initial gate's `(id(response), index)` keying. + occurrences = {id(call): index for index, call in enumerate(calls)} + for call in initial_survivors: + tool = self._find_tool(call.name, tools) + if tool is None or not is_auto_resolvable_tool(tool): + continue + gate_key = (id(response), occurrences.get(id(call), call.id)) + if await self._prepare_after_initial_approval(call, turn_context, tools, gate_key) == "pending": + prepared = self._prepared_tool_calls.get(call.id) + if prepared is not None and prepared["type"] == "ready": + still_pending.append(prepared["call"]) + return denied, still_pending, initial_survivors + + async def _classify_initial_approval_calls( + self, calls: Sequence[ParsedToolCall], response: Any, tools: Sequence[Tool] + ) -> Tuple[List[ParsedToolCall], List[ParsedToolCall]]: + """Mark each not-yet-gated call occurrence as gated and run the + doom-loop checkpoint on it (upstream `classifyInitialApprovalCalls`). + + The checkpoint runs on the model-issued arguments BEFORE approval + partitioning, PermissionRequest, and PreToolUse. Returns `(unseen, + blocked)`: `blocked` calls were refused by the checkpoint and are + excluded from partitioning; `_run_tool_with_hooks` later synthesizes + their error output without executing them. + """ + response_key = id(response) + unseen_candidates = [ + call for index, call in enumerate(calls) if (response_key, index) not in self._completed_approval_gates + ] + await self._begin_doom_loop_round(unseen_candidates, tools) + + unseen: List[ParsedToolCall] = [] + blocked: List[ParsedToolCall] = [] + for index, call in enumerate(calls): + gate = (response_key, index) + if gate in self._completed_approval_gates: + continue + self._completed_approval_gates.add(gate) + tool = self._find_tool(call.name, tools) + if self._doom_loop_monitor is not None and tool is not None and is_auto_resolvable_tool(tool): + if isinstance(call.arguments, str): + # Malformed call: its identity is the raw string. + decision = await self._evaluate_doom_loop_call( + tool_name=str(call.name), + key_material=call.arguments, + allow_block=True, + detector="tool-fingerprint", + call=call, + ) + is_blocked = decision["action"] in ("block", "stop") + reason = decision.get("message") or "Blocked by doom loop" + else: + is_blocked, reason = await self._check_doom_loop_before_execution(tool, call) + if is_blocked: + self._prepared_tool_calls[call.id] = {"type": "blocked", "reason": reason} + blocked.append(call) + continue + unseen.append(call) + return unseen, blocked + + async def _process_approval_decisions(self, request: Mapping[str, Any]) -> Optional[Dict[str, Any]]: + """Apply approve/reject decisions to a paused conversation. + + Returns the resume request, or None when the conversation stays paused + (some pending calls still owe a decision, or an approved HITL call + paused again). + """ + self._fresh_items_for_state = [] + state = self._state + assert state is not None + tools: Sequence[Tool] = self.options.get("tools") or [] + pending: List[ParsedToolCall] = list(state.pending_tool_calls or []) + unsent: List[UnsentToolResult] = list(state.unsent_tool_results or []) + self._call_names.update({call.id: call.name for call in pending}) + hitl_paused: set[str] = set() + # Calls that pause again after a PreToolUse mutation stay pending, + # marked so the next resume does not re-run the hook. + second_gate_paused: set[str] = set() + turn_context = {"number_of_turns": self._all_tool_rounds + 1} + persisted_key = f"persisted:{state.previous_response_id or 'unknown'}" + + for call_id in self._approved_ids: + index = next((i for i, c in enumerate(pending) if c.id == call_id), None) + if index is None: + continue + call = pending[index] + tool = self._find_tool(call.name, tools) + if not tool or not is_auto_resolvable_tool(tool): + unsent.append(create_rejected_result(call_id, call.name, "Tool not found or not executable")) + continue + if not call.pre_tool_use_applied: + # Approved on the model-issued arguments (or legacy state + # without the marker): PreToolUse runs now, and a mutated + # input needs its own approval. + prepared = await self._prepare_after_initial_approval( + call, turn_context, tools, (persisted_key, call_id) + ) + if prepared == "pending": + ready = self._prepared_tool_calls.get(call_id) + if ready is not None and ready["type"] == "ready": + pending[index] = dataclasses.replace(ready["call"], pre_tool_use_applied=True) + second_gate_paused.add(call_id) + continue + outcome = await self._run_tool_with_hooks(tool, call, turn_context, run_pre_tool_use=False) + if outcome["type"] == "parse_error": + await self._append_tool_result_event(call, tool, {"error": outcome["error_message"]}) + unsent.append(create_rejected_result(call_id, call.name, outcome["error_message"])) + elif outcome["type"] == "hook_blocked": + unsent.append(create_rejected_result(call_id, call.name, outcome["reason"])) + elif outcome["result"] is None: + # HITL tool paused on approval -- keep it visible on + # pending_tool_calls (status becomes awaiting_hitl below). + hitl_paused.add(call_id) + elif outcome["result"].get("error") is not None: + unsent.append(create_rejected_result(call_id, call.name, str(outcome["result"]["error"]))) + else: + unsent.append(create_unsent_result(call_id, call.name, outcome["result"].get("result"))) + + for call_id in self._rejected_ids: + rejected = next((c for c in pending if c.id == call_id), None) + if rejected is None: + continue + call = rejected + reason = "Rejected by user" + if call.pre_tool_use_applied: + # The call's lifecycle already entered PreToolUse (it paused + # on the mutated-input gate), so the rejection is a failure. + await self._emit_prepared_failure(call, reason) + self._prepared_tool_calls.pop(call_id, None) + self._hook_denied_calls.pop(call_id, None) + unsent.append(create_rejected_result(call_id, call.name, reason)) + + processed = { + cid + for cid in [*self._approved_ids, *self._rejected_ids] + if cid not in hitl_paused and cid not in second_gate_paused + } + remaining = [c for c in pending if c.id not in processed] + remaining_unresolved = [c for c in remaining if c.id not in hitl_paused] + if remaining_unresolved: + next_status = "awaiting_approval" + elif hitl_paused: + next_status = "awaiting_hitl" + else: + next_status = "in_progress" + if next_status == "in_progress" and self._doom_loop_stop is not None: + # This approval continuation is terminal: no model request + # follows, so a later fresh run must not inherit its consumed + # caller choice. + self._reset_forced_tool_choice_policy() + await self._save_state( + status=next_status, + pending_tool_calls=remaining or None, + unsent_tool_results=unsent or None, + ) + if next_status != "in_progress": + # Queued steer guidance rides doom_loop.pending_steer in the + # state persisted above; the resumed run delivers it later. + self._prepared_tool_calls.clear() + self._hook_denied_calls.clear() + return None + + # A doom stop re-armed from persisted state (decision-only resume of + # a condemned run): halt WITHOUT the unsent-results model request. + # The executed outputs stay persisted as unsent results. + if self._doom_loop_stop is not None: + return None + + # Continue with the unsent results. SDK-generated outputs are + # appended as-is: `on_response_received` is reserved for + # caller-supplied outputs. + outputs = unsent_results_to_api_format(unsent) + updated_messages = append_to_messages(state.messages, outputs) + await self._save_state(messages=updated_messages, unsent_tool_results=None) + return {**dict(request), "input": updated_messages, "stream": True} + + async def _append_tool_result_event(self, call: ParsedToolCall, tool: Tool, result: Any) -> None: + await self._append_event( + { + "type": "tool.result", + "toolCallId": call.id, + "tool_call_id": call.id, + "toolName": call.name, + "tool_name": call.name, + "source": "mcp" if is_mcp_tool(tool) else "client", + "result": result, + "timestamp": _now_ms(), + } + ) + + # -- tool rounds ------------------------------------------------------------ + + async def _execute_round( + self, + calls: Sequence[ParsedToolCall], + tools: Sequence[Tool], + turn_context: Mapping[str, Any], + step: StepResult, + ) -> Tuple[List[Any], List[ParsedToolCall], List[ParsedToolCall]]: + """Execute one round of tool calls. Returns `(outputs, paused, + executed_calls)`. Calls denied by a PermissionRequest hook get a + synthesized rejected output instead of executing.""" + outputs: List[Any] = [] + paused: List[ParsedToolCall] = [] + executed_calls: List[ParsedToolCall] = [] + for call in calls: + denied_reason = self._hook_denied_calls.pop(call.id, None) + if denied_reason is not None: + output = self._rejected_output(call.id, denied_reason) + outputs.append(output) + self._tool_outputs.append(output) + await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) + continue + tool = self._find_tool(call.name, tools) + if not tool or not is_auto_resolvable_tool(tool): + continue + outcome = await self._run_tool_with_hooks(tool, call, turn_context) + if outcome["type"] == "parse_error": + await self._append_tool_result_event(call, tool, {"error": outcome["error_message"]}) + output = self._rejected_output(call.id, outcome["error_message"]) + outputs.append(output) + self._tool_outputs.append(output) + await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) + continue + if outcome["type"] == "hook_blocked": + output = self._rejected_output(call.id, outcome["reason"]) + outputs.append(output) + self._tool_outputs.append(output) + await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) + continue + + effective_call = outcome["call"] + result = outcome["result"] + if result is None: + paused.append(call) + continue + await self._append_event( + { + "type": "tool.result", + "toolCallId": call.id, + "tool_call_id": call.id, + "toolName": call.name, + "tool_name": call.name, + "source": result.get("source", "mcp" if is_mcp_tool(tool) else "client"), + "result": {"error": str(result["error"])} + if result.get("error") is not None + else result.get("result"), + "timestamp": _now_ms(), + } + ) + output = await self._tool_result_to_output(effective_call, tool, result) + outputs.append(output) + executed_calls.append(effective_call) + self._tool_outputs.append(output) + await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) + step.tool_results.append(result) + return outputs, paused, executed_calls + + async def _resolve_request_for_turn(self, current_request: Dict[str, Any], number_of_turns: int) -> Dict[str, Any]: + """Re-resolve async/callable request params for a turn, preserving the + accumulated input (upstream `resolveAsyncFunctionsForTurn`).""" + if not has_async_functions(self.options["request"]): + return current_request + resolved = await resolve_async_functions(self.options["request"], {"number_of_turns": number_of_turns}) + resolved["stream"] = True + if "input" in current_request: + resolved["input"] = current_request["input"] + return self._apply_resolved_forced_tool_choice_policy(resolved) + # -- the main loop ----------------------------------------------------- async def _run(self) -> Any: @@ -589,9 +1883,22 @@ async def _run(self) -> Any: self._session_id = self._state.id if self._state is not None else generate_conversation_id() tools: Sequence[Tool] = self.options.get("tools") or [] + request = await resolve_async_functions(self.options["request"], {"number_of_turns": 0}) + + # Paused for a human decision but none was supplied: stay paused. + # No request is sent and the supplied input is neither sent nor + # saved; every response/stream getter raises an explicit pause + # error (see `_paused_error`). + if self._stays_paused_without_decisions(request.get("input")): + return None + + decision_resume = self._is_decision_resume() + if self._state is not None and not self._resuming_from_client_tools and not decision_resume: + await self._save_state(status="in_progress") + await self._emit_session_start(tools) - request = await resolve_async_functions(self.options["request"], {"number_of_turns": 0}) + request = self._apply_resolved_forced_tool_choice_policy(request) request["stream"] = True base_input = request.get("input") @@ -620,22 +1927,28 @@ async def _run(self) -> Any: fresh_input = hooked_array[len(historical_function_calls) :] if self._state is not None: request["input"] = append_to_messages(historical_input, fresh_input) - if not ( - self._state.pending_tool_calls - and (self.options.get("approve_tool_calls") or self.options.get("reject_tool_calls")) - ): + if not decision_resume: self._fresh_items_for_state = list(fresh_input) elif fresh_input: request["input"] = fresh_input current_request = request is_resume_turn = self._resuming_from_client_tools - if ( - self._state is not None - and self._state.pending_tool_calls - and (self.options.get("approve_tool_calls") or self.options.get("reject_tool_calls")) - ): - current_request = await self._build_resume_request(request) + if decision_resume: + resumed = await self._process_approval_decisions(request) + if resumed is None: + # A doom stop on an approval continuation: like a pause, no + # response exists, but the session ends as doom-stopped. + if ( + self._doom_loop_stop is not None + and self._state is not None + and self._state.status == "in_progress" + ): + session_end_reason = "doom_loop" + self._reset_forced_tool_choice_policy() + await self._save_state() + return None + current_request = resumed is_resume_turn = True final_response = None @@ -661,7 +1974,14 @@ async def _run(self) -> Any: turn_type = "tool_round" pending_final_directive = False - response = await self._send_and_track(current_request, turn_type, turn_number) + # Escalation recovery: one-turn overrides (model swap / forced + # advisor consult) applied to a tool-round follow-up dispatch + # only; `current_request` keeps the configured model/tools, so + # the next turn reverts automatically. + dispatch_request = ( + self._apply_doom_loop_escalation(current_request) if turn_type == "tool_round" else current_request + ) + response = await self._send_and_track(dispatch_request, turn_type, turn_number) await self._append_event( { "type": "turn.end", @@ -674,6 +1994,7 @@ async def _run(self) -> Any: await maybe_await(self.options["on_turn_end"]({"number_of_turns": turn_number}, response)) calls = extract_tool_calls_from_response(response) self._tool_calls.extend(calls) + self._call_names.update({call.id: call.name for call in calls}) final_response = response await self._save_response_to_state(response) text = extract_text_from_response(response) @@ -686,14 +2007,52 @@ async def _run(self) -> Any: usage=get_field(response, "usage"), ) self._steps.append(step) + # The forced final turn (tool_choice "none") ends the run: any + # tool calls it still carries are not executed. + if turn_type == "final": + break + # Doom-loop text + server-tool checkpoint for the fresh response. + await self._check_doom_loop_for_response(response) if not calls or not tools: - await self._save_state(status="complete") + # A text verdict may have armed a stop: report the run as + # doom-stopped, not as a normal completion. + if self._doom_loop_stop is not None: + session_end_reason = "doom_loop" + await self._mark_state_complete() break stop_when = self.options.get("stop_when") stop_conditions = list(stop_when) if isinstance(stop_when, list) else ([stop_when] if stop_when else []) + + if turn_number == 0: + # With allow_final_response=False, a met stop_when means no + # tool from this response runs. Evaluate it before the + # approval check so pausing for an approval-gated call does + # not execute its auto-approved siblings. + skip_auto_execution = ( + self.options.get("allow_final_response") is False + and bool(stop_conditions) + and await is_stop_condition_met(stop_conditions, self._executed_rounds) + ) + if await self._handle_approval_check(calls, 0, response, tools, skip_auto_execution): + return final_response + # All calls manual / not auto-resolvable: surface them as + # pending client tools instead of completing. + if not any(self._call_is_auto_resolvable(c, tools) for c in calls): + await self._persist_client_tools_pause(response, calls) + return final_response + + # A doom-loop 'stop' verdict armed by this response's text (or + # an earlier checkpoint) halts the run before any further + # spend. Seal first: this response's calls have not executed, + # so they need synthesized halt outputs. + if self._doom_loop_stop is not None: + await self._seal_doom_loop_stop(response) + session_end_reason = "doom_loop" + break + stopped_by_stop_when = False - while stop_conditions and await is_stop_condition_met(stop_conditions, self._steps): + while stop_conditions and await is_stop_condition_met(stop_conditions, self._executed_rounds): stop_decision = await self._run_stop_hook(force_resume_count, current_request) if stop_decision == "resume": # Zero-cost retry: re-check the stop condition against @@ -709,177 +2068,68 @@ async def _run(self) -> Any: stopped_by_stop_when = True break + round_context = {"number_of_turns": turn_number + 1, "turn_request": current_request} + if stopped_by_stop_when: + # stop_when broke the loop while the model was still + # emitting tool calls: execute them so they have matching + # outputs, then make one more tool_choice "none" request to + # coerce a final text response. Default-on. allow_final_response = self.options.get("allow_final_response") final_response_enabled = allow_final_response is not False - resolvable_pending = [c for c in calls if self._call_is_auto_resolvable(c, tools)] - if final_response_enabled and resolvable_pending: - final_outputs: List[Any] = [] - turn_context = {"number_of_turns": turn_number + 1, "turn_request": current_request} - for call in calls: - matching_tool = self._find_tool(call.name, tools) - if matching_tool and is_auto_resolvable_tool(matching_tool): - outcome = await self._run_tool_with_hooks(matching_tool, call, turn_context) - if outcome["type"] == "parse_error": - final_outputs.append(self._rejected_output(call.id, outcome["error_message"])) - elif outcome["type"] == "hook_blocked": - final_outputs.append(self._rejected_output(call.id, outcome["reason"])) - else: - result = outcome["result"] - if result is not None: - output = await self._tool_result_to_output( - outcome["call"], matching_tool, result - ) - final_outputs.append(output) - self._tool_outputs.append(output) - await self._append_event( - { - "type": "tool.call_output", - "output": output, - "timestamp": _now_ms(), - } - ) - else: - final_outputs.append( - { - "type": "function_call_output", - "id": f"output_{call.id}", - "callId": call.id, - "output": "Tool execution skipped: step limit reached.", - } - ) - await self._save_tool_outputs_to_state(final_outputs) + if final_response_enabled and any(self._call_is_auto_resolvable(c, tools) for c in calls): + # Gate these calls exactly like a normal round would: + # this path executes real tools. + if await self._handle_approval_check(calls, turn_number + 1, response, tools): + return final_response + current_request = await self._resolve_request_for_turn(current_request, turn_number + 1) + outputs, paused, executed_calls = await self._execute_round(calls, tools, round_context, step) + self._record_executed_round(calls, response, outputs, tools) + # Persist real outputs BEFORE the HITL pause check so a + # partial batch keeps the regular tool's output. + await self._save_tool_outputs_to_state(outputs) + if paused: + await self._save_state(pending_tool_calls=paused, status="awaiting_hitl") + self._final_response = final_response + return final_response + self._all_tool_rounds += 1 + # A doom stop armed while executing the halted turn's + # calls: seal and skip the final text-coercion request + # ("no further model requests after stop"). + if self._doom_loop_stop is not None: + await self._seal_doom_loop_stop(response, {get_field(o, "callId") for o in outputs}) + session_end_reason = "doom_loop" + break + await self._apply_next_turn_params(executed_calls, tools, current_request) + current_request = self._pending_next_request or current_request + self._pending_next_request = None + # Pair any unexecuted call with a stub output so every + # function_call in the *request* has a matching output. + # Stubs are NOT persisted to state. + executed_ids = {get_field(o, "callId") for o in outputs} + stubs = [ + { + "type": "function_call_output", + "id": f"output_{call.id}", + "callId": call.id, + "output": "Tool execution skipped: step limit reached.", + } + for call in calls + if call.id not in executed_ids + ] current_request = self._build_final_request( - current_request, response, final_outputs, allow_final_response + current_request, response, [*outputs, *stubs], allow_final_response ) pending_final_directive = True continue break - partition = await partition_tool_calls( - calls, tools, {"number_of_turns": turn_number + 1}, self.options.get("require_approval") - ) - requires_approval = list(partition["requires_approval"]) - hook_resolved_unsent: List[UnsentToolResult] = [] - if requires_approval and hooks: - still_pending: List[ParsedToolCall] = [] - for call in requires_approval: - decision, reason = await self._emit_permission_request(call, tools) - if decision == "allow": - promo_tool = self._find_tool(call.name, tools) - if promo_tool and is_auto_resolvable_tool(promo_tool): - outcome = await self._run_tool_with_hooks( - promo_tool, - call, - { - "number_of_turns": turn_number + 1, - "tool_call": call, - "turn_request": current_request, - }, - ) - if outcome["type"] == "parse_error": - hook_resolved_unsent.append( - create_rejected_result(call.id, call.name, outcome["error_message"]) - ) - elif outcome["type"] == "hook_blocked": - hook_resolved_unsent.append( - create_rejected_result(call.id, call.name, outcome["reason"]) - ) - elif outcome["result"] is None: - still_pending.append(call) - elif outcome["result"].get("error") is not None: - hook_resolved_unsent.append( - create_rejected_result(call.id, call.name, str(outcome["result"]["error"])) - ) - else: - hook_resolved_unsent.append( - create_unsent_result(call.id, call.name, outcome["result"].get("result")) - ) - else: - still_pending.append(call) - elif decision == "deny": - hook_resolved_unsent.append( - create_rejected_result(call.id, call.name, reason or "Denied by PermissionRequest hook") - ) - else: - still_pending.append(call) - requires_approval = still_pending - - if requires_approval: - if self.options.get("state") is None: - names = ", ".join(call.name for call in requires_approval) - raise ValueError(f"Tool(s) require approval but no state accessor is configured: {names}") - save_kwargs: Dict[str, Any] = { - "pending_tool_calls": requires_approval, - "status": "awaiting_approval", - } - if hook_resolved_unsent: - save_kwargs["unsent_tool_results"] = hook_resolved_unsent - await self._save_state(**save_kwargs) - self._final_response = final_response + if await self._handle_approval_check(calls, turn_number + 1, response, tools): return final_response - outputs: List[Any] = unsent_results_to_api_format(hook_resolved_unsent) if hook_resolved_unsent else [] - paused: List[ParsedToolCall] = [] - executed_calls: List[ParsedToolCall] = [] - for call in partition["auto_execute"]: - tool = self._find_tool(call.name, tools) - if not tool or not is_auto_resolvable_tool(tool): - continue - turn_context = { - "number_of_turns": turn_number + 1, - "tool_call": call, - "turn_request": current_request, - } - outcome = await self._run_tool_with_hooks(tool, call, turn_context) - if outcome["type"] == "parse_error": - await self._append_event( - { - "type": "tool.result", - "toolCallId": call.id, - "tool_call_id": call.id, - "source": "mcp" if is_mcp_tool(tool) else "client", - "result": {"error": outcome["error_message"]}, - "timestamp": _now_ms(), - } - ) - output = self._rejected_output(call.id, outcome["error_message"]) - outputs.append(output) - self._tool_outputs.append(output) - await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) - continue - if outcome["type"] == "hook_blocked": - output = self._rejected_output(call.id, outcome["reason"]) - outputs.append(output) - self._tool_outputs.append(output) - await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) - continue - - effective_call = outcome["call"] - result = outcome["result"] - if result is None: - paused.append(call) - continue - await self._append_event( - { - "type": "tool.result", - "toolCallId": call.id, - "tool_call_id": call.id, - "source": result.get("source", "mcp" if is_mcp_tool(tool) else "client"), - "result": {"error": str(result["error"])} - if result.get("error") is not None - else result.get("result"), - "preliminaryResults": result.get("preliminary_results"), - "preliminary_results": result.get("preliminary_results"), - "timestamp": _now_ms(), - } - ) - output = await self._tool_result_to_output(effective_call, tool, result) - outputs.append(output) - executed_calls.append(effective_call) - self._tool_outputs.append(output) - await self._append_event({"type": "tool.call_output", "output": output, "timestamp": _now_ms()}) - step.tool_results.append(result) + current_request = await self._resolve_request_for_turn(current_request, turn_number + 1) + outputs, paused, executed_calls = await self._execute_round(calls, tools, round_context, step) + self._record_executed_round(calls, response, outputs, tools) if paused: await self._save_tool_outputs_to_state(outputs) @@ -895,13 +2145,28 @@ async def _run(self) -> Any: await self._persist_client_tools_pause(response, unresolved_calls) return final_response + # A doom stop armed during this round halts before the next + # model request. Seal defensively: the condemned call already + # has a blocked output, but other calls may be unresolved. + if self._doom_loop_stop is not None: + if outputs: + self._all_tool_rounds += 1 + await self._seal_doom_loop_stop(response, resolved_ids) + session_end_reason = "doom_loop" + break + if not outputs: break self._all_tool_rounds += 1 force_resume_count = 0 - next_params = await execute_next_turn_params_functions(executed_calls, tools, current_request) - if next_params: - current_request = apply_next_turn_params_to_request(current_request, next_params) + # Inject queued doom-loop steer guidance so the very next + # request carries it. It lands before this round's call/output + # items (the injection point predates the follow-up + # accumulation) -- matching upstream. + await self._flush_doom_loop_steer(current_request) + await self._apply_next_turn_params(executed_calls, tools, current_request) + current_request = self._pending_next_request or current_request + self._pending_next_request = None current_request = self._build_followup_request(current_request, response, outputs) else: raise RuntimeError("call_model exceeded the 20-turn safety limit") @@ -924,7 +2189,7 @@ async def _run(self) -> Any: allow_empty_output = can_tolerate_empty and is_empty_output self._validate_final_response(final_response, allow_empty_output) self._final_response = final_response - await self._save_state(status="complete") + await self._mark_state_complete() return final_response except Exception: session_end_reason = "error" @@ -940,40 +2205,62 @@ async def _run(self) -> Any: self._run_done = True self._condition.notify_all() + def _record_executed_round( + self, calls: Sequence[ParsedToolCall], response: Any, outputs: Sequence[Any], tools: Sequence[Tool] + ) -> None: + """Record an executed tool round as a stop-condition step. `tool_results` + is client-tool-centric; server-tool output items are surfaced on + `server_tool_results` so conditions can react to either class.""" + tool_results: List[Dict[str, Any]] = [] + for output in outputs: + call_id = get_field(output, "callId") + name = next((c.name for c in calls if c.id == call_id), "") + matched = self._find_tool(name, tools) + raw = get_field(output, "output") + tool_results.append( + { + "tool_call_id": call_id, + "tool_name": name, + "source": "mcp" if matched is not None and is_mcp_tool(matched) else "client", + "result": json_loads_maybe(raw) if isinstance(raw, str) else raw, + } + ) + server_items = [ + item + for item in self._response_output_items(response) + if get_field(item, "type") not in (None, "message", "reasoning", "function_call") + ] + self._executed_rounds.append( + StepResult( + step_type="continue", + text=extract_text_from_response(response), + tool_calls=list(calls), + tool_results=tool_results, + response=response, + server_tool_results=server_items, + usage=get_field(response, "usage"), + ) + ) + + async def _apply_next_turn_params( + self, executed_calls: Sequence[ParsedToolCall], tools: Sequence[Tool], current_request: Dict[str, Any] + ) -> None: + """Apply `next_turn_params` from executed tools. A tool-computed + `tool_choice` becomes the new caller-level policy, not a one-turn + override, so it re-runs the resolved-policy bookkeeping.""" + self._pending_next_request = None + next_params = await execute_next_turn_params_functions(executed_calls, tools, current_request) + if not next_params: + return + next_request = apply_next_turn_params_to_request(current_request, next_params) + if "tool_choice" in next_params: + next_request = self._apply_resolved_forced_tool_choice_policy(next_request) + self._pending_next_request = next_request + def _call_is_auto_resolvable(self, call: ParsedToolCall, tools: Sequence[Tool]) -> bool: tool = self._find_tool(call.name, tools) return bool(tool and is_auto_resolvable_tool(tool)) - async def _build_resume_request(self, request: Mapping[str, Any]) -> Dict[str, Any]: - self._fresh_items_for_state = [] - approved = set(self.options.get("approve_tool_calls") or []) - rejected = set(self.options.get("reject_tool_calls") or []) - outputs: List[Any] = [] - if self._state is not None and self._state.unsent_tool_results: - outputs.extend(unsent_results_to_api_format(self._state.unsent_tool_results)) - pending: List[ParsedToolCall] = self._state.pending_tool_calls if self._state is not None else [] - for call in pending or []: - tool = self._find_tool(call.name, self.options.get("tools") or []) - if call.id in rejected: - outputs.append(self._rejected_output(call.id, "Tool call rejected by user")) - elif call.id in approved and tool and is_auto_resolvable_tool(tool): - turn_context = {"number_of_turns": 0} - outcome = await self._run_tool_with_hooks(tool, call, turn_context) - if outcome["type"] == "parse_error": - outputs.append(self._rejected_output(call.id, outcome["error_message"])) - elif outcome["type"] == "hook_blocked": - outputs.append(self._rejected_output(call.id, outcome["reason"])) - elif outcome["result"] is not None: - outputs.append(await self._tool_result_to_output(outcome["call"], tool, outcome["result"])) - updated_messages = append_to_messages(self._state.messages, outputs) if self._state is not None else outputs - await self._save_state( - messages=updated_messages, - pending_tool_calls=None, - unsent_tool_results=None, - status="in_progress", - ) - return {**dict(request), "input": updated_messages, "stream": True} - async def _iter_events_live(self) -> AsyncIterator[Any]: self._ensure_task() index = 0 @@ -997,11 +2284,15 @@ def _build_followup_request( output_items = get_field(response, "output", []) or [] if not isinstance(output_items, list): output_items = [output_items] - return { + followup = { **dict(request), "input": [*normalize_input_to_array(request.get("input")), *output_items, *outputs], "stream": True, } + # A forced tool choice has served its purpose once a tool round has + # executed; keeping it would forbid the model from ever answering in + # text (DEV-785). Relax it so the model can call again or answer. + return self._apply_forced_tool_choice_policy(followup) def _build_final_request( self, request: Mapping[str, Any], response: Any, outputs: Sequence[Any], allow_final_response: Any @@ -1036,7 +2327,35 @@ async def _retry_current_request(self, current_request: Mapping[str, Any], turn_ # -- public streaming/result surface ------------------------------------ async def get_response(self) -> Any: - return await self._ensure_task() + """The final round's response. Raises an explicit pause error when a + resume left the conversation paused without producing a response.""" + await self._ensure_task() + if self._final_response is None: + raise self._response_unavailable_error() + return self._final_response + + async def get_doom_loop_verdict(self) -> Optional[DoomLoopVerdict]: + """The doom-loop verdict that stopped this run, or None when the run + was not stopped by doom-loop detection (including when detection is + off). Resolves after the tool loop settles. For per-event observation + use the `DoomLoopDetected` hook.""" + await self._ensure_task() + return self._doom_loop_stop + + async def get_usage(self) -> Dict[str, Any]: + """Aggregate token/cost usage across every model call this run made + (initial request, each tool-round follow-up, the empty-final retry, + the `allow_final_response` final turn, and approval-resume requests). + + Same shape as the `SessionEnd` hook's `total_usage`. Never raises: a + failed run still consumed tokens, so the totals accrued so far are + returned. `cost` is present only when the server reported it. + """ + try: + await self._ensure_task() + except Exception as error: # noqa: BLE001 - documented never-raises accessor + warnings.warn(f"[get_usage] run failed; reporting totals accrued so far: {error}", stacklevel=2) + return self._snapshot_session_usage() async def get_text(self) -> str: response = await self.get_response() @@ -1081,6 +2400,8 @@ async def get_tool_stream(self) -> AsyncIterator[Dict[str, Any]]: "type": "preliminary_result", "toolCallId": get_field(event, "toolCallId"), "tool_call_id": get_field(event, "tool_call_id", get_field(event, "toolCallId")), + "toolName": get_field(event, "toolName"), + "tool_name": get_field(event, "tool_name", get_field(event, "toolName")), "result": get_field(event, "result"), } elif typ == "tool.result": @@ -1088,10 +2409,10 @@ async def get_tool_stream(self) -> AsyncIterator[Dict[str, Any]]: "type": "tool_result", "toolCallId": get_field(event, "toolCallId"), "tool_call_id": get_field(event, "tool_call_id", get_field(event, "toolCallId")), + "toolName": get_field(event, "toolName"), + "tool_name": get_field(event, "tool_name", get_field(event, "toolName")), "source": get_field(event, "source", "client"), "result": get_field(event, "result"), - "preliminaryResults": get_field(event, "preliminaryResults"), - "preliminary_results": get_field(event, "preliminary_results"), } elif typ == "tool.call_output": yield {"type": "tool_call_output", "output": get_field(event, "output")} @@ -1208,7 +2529,7 @@ async def get_full_chat_stream(self) -> AsyncIterator[Dict[str, Any]]: yield {"type": "message.complete", "response": await self.get_response()} async def requires_approval(self) -> bool: - await self.get_response() + await self._ensure_task() state = self._state return bool( state @@ -1219,11 +2540,11 @@ async def requires_approval(self) -> bool: ) async def get_pending_tool_calls(self) -> List[ParsedToolCall]: - await self.get_response() + await self._ensure_task() return list(self._state.pending_tool_calls or []) if self._state else [] async def get_state(self) -> Any: - await self.get_response() + await self._ensure_task() return self._state def cancel(self) -> None: diff --git a/src/openrouter_agent/next_turn_params.py b/src/openrouter_agent/next_turn_params.py index 2c670e5..0114bd8 100644 --- a/src/openrouter_agent/next_turn_params.py +++ b/src/openrouter_agent/next_turn_params.py @@ -1,32 +1,90 @@ from __future__ import annotations +import warnings from typing import Any, Dict, Mapping, Sequence from ._utils import maybe_await from .tool_types import ParsedToolCall, Tool, get_tool_function, is_client_tool -NEXT_TURN_KEYS = ("input", "model", "models", "temperature", "max_output_tokens", "top_p", "top_k", "instructions") +NEXT_TURN_KEYS = ( + "input", + "tool_choice", + "model", + "models", + "temperature", + "max_output_tokens", + "top_p", + "top_k", + "instructions", +) def build_next_turn_params_context(request: Mapping[str, Any]) -> Dict[str, Any]: - return {key: request.get(key) for key in NEXT_TURN_KEYS if key in request} + """Current parameter values handed to `next_turn_params` functions, with + upstream's defaults for absent fields. `tool_choice` and `top_k` are + passed through only when present (upstream leaves them `undefined`).""" + context: Dict[str, Any] = { + "input": request.get("input") if request.get("input") is not None else [], + "model": request.get("model") or "", + "models": request.get("models") or [], + "temperature": request.get("temperature"), + "max_output_tokens": request.get("max_output_tokens"), + "top_p": request.get("top_p"), + "instructions": request.get("instructions"), + } + for optional_key in ("tool_choice", "top_k"): + if request.get(optional_key) is not None: + context[optional_key] = request[optional_key] + return context + + +def _describe_type(value: Any) -> str: + if isinstance(value, (list, tuple)): + return "array" + if isinstance(value, str): + return "string" + if value is None: + return "object" + if isinstance(value, bool): + return "boolean" + if isinstance(value, (int, float)): + return "number" + return "object" async def execute_next_turn_params_functions( tool_calls: Sequence[ParsedToolCall], tools: Sequence[Tool], request: Mapping[str, Any] ) -> Dict[str, Any]: + """Execute `next_turn_params` functions for all called tools, in tools + order, composing values when several tools modify the same parameter + (later tools see earlier tools' results).""" context = build_next_turn_params_context(request) computed: Dict[str, Any] = {} - for call in tool_calls: - matching = next( - (tool for tool in tools if is_client_tool(tool) and get_tool_function(tool).get("name") == call.name), None - ) - if not matching: + for matching in tools: + if not is_client_tool(matching): + continue + fn_def = get_tool_function(matching) + next_params = fn_def.get("next_turn_params") + if not next_params: continue - fns = get_tool_function(matching).get("next_turn_params") or {} - for key, fn in fns.items(): - computed[key] = await maybe_await(fn(call.arguments, context)) - context[key] = computed[key] + name = fn_def.get("name") + for call in [c for c in tool_calls if c.name == name]: + if not isinstance(call.arguments, Mapping): + raise TypeError( + f"Tool call arguments for {name} must be an object, got {_describe_type(call.arguments)}" + ) + for key, fn in next_params.items(): + if not callable(fn): + continue + if key not in NEXT_TURN_KEYS: + warnings.warn( + f'Invalid next_turn_params key "{key}" in tool "{name}". ' + f"Valid keys: {', '.join(NEXT_TURN_KEYS)}", + stacklevel=2, + ) + continue + computed[key] = await maybe_await(fn(call.arguments, context)) + context[key] = computed[key] return computed diff --git a/src/openrouter_agent/openrouter.py b/src/openrouter_agent/openrouter.py index b799025..d516b99 100644 --- a/src/openrouter_agent/openrouter.py +++ b/src/openrouter_agent/openrouter.py @@ -1,15 +1,15 @@ from __future__ import annotations import inspect -from importlib import metadata -from typing import Any, Callable, Mapping, Optional, Protocol +from typing import Any, Mapping, Optional, Protocol from openrouter import OpenRouter as _SDKOpenRouter from openrouter._hooks import SDKHooks from openrouter._version import __user_agent__ - from .call_model import call_model +from .user_agent import AGENT_PACKAGE_NAME as AGENT_PACKAGE_NAME +from .user_agent import agent_user_agent_token, append_agent_user_agent class Hook(Protocol): @@ -19,25 +19,10 @@ def sdk_init(self, configuration: Any) -> Any: ... OpenRouterOptions = Mapping[str, object] SDKOptions = Mapping[str, object] -AGENT_PACKAGE_NAME = "openrouter-agent-sdk" - - -def _agent_user_agent_token( - version_getter: Optional[Callable[[str], str]] = None, -) -> str: - try: - version = (version_getter or metadata.version)(AGENT_PACKAGE_NAME) - except metadata.PackageNotFoundError: - return AGENT_PACKAGE_NAME - return f"{AGENT_PACKAGE_NAME}/{version}" - - -def _append_agent_user_agent(user_agent: str, agent_token: str) -> str: - if agent_token in user_agent.split(): - return user_agent - if not user_agent: - return agent_token - return f"{user_agent} {agent_token}" +# Kept as module-level names for backwards compatibility; the implementation +# lives in user_agent.py (upstream lib/user-agent.ts). +_agent_user_agent_token = agent_user_agent_token +_append_agent_user_agent = append_agent_user_agent class OpenRouter(_SDKOpenRouter): # type: ignore[misc, valid-type] @@ -58,9 +43,12 @@ def __init__(self, *args: Any, hooks: Any = None, **kwargs: Any) -> None: if explicit_user_agent is not None: self.sdk_configuration.user_agent = explicit_user_agent elif self.sdk_configuration.user_agent == __user_agent__: - self.sdk_configuration.user_agent = _append_agent_user_agent( + # Upstream: `userAgent: rest.userAgent ?? AGENT_USER_AGENT`. The token + # is resolved per construction (not the import-time AGENT_USER_AGENT) + # so it always reflects the installed distribution metadata. + self.sdk_configuration.user_agent = append_agent_user_agent( self.sdk_configuration.user_agent, - _agent_user_agent_token(), + agent_user_agent_token(), ) @staticmethod diff --git a/src/openrouter_agent/reusable_stream.py b/src/openrouter_agent/reusable_stream.py index 5fb982a..f1f94ac 100644 --- a/src/openrouter_agent/reusable_stream.py +++ b/src/openrouter_agent/reusable_stream.py @@ -1,22 +1,288 @@ +"""Replayable fan-out over a single async source. + +Ports `packages/agent/src/lib/reusable-stream.ts`. + +Key features: + +- Multiple concurrent consumers with independent read positions +- New consumers can attach while streaming is active +- Full replay for delayed and sequential consumers by default +- Opt-in active-consumer replay compaction for bounded memory + (``stream_replay="active-consumers"``) +- Each consumer can read at its own pace +""" + from __future__ import annotations import asyncio -from typing import Any, AsyncIterable, AsyncIterator, List, Optional +from typing import ( + Any, + AsyncIterable, + AsyncIterator, + Callable, + Dict, + Generic, + List, + Literal, + Optional, + TypeVar, + Union, +) +T = TypeVar("T") -class ReusableReadableStream: - def __init__(self, source: AsyncIterable[Any]) -> None: - self._source = source +StreamReplay = Literal["full", "active-consumers"] +"""Replay policy shared by `ReusableReadableStream` and `ToolEventBroadcaster`. + +``"full"`` (default) retains every value so delayed and sequential consumers +replay the complete history. ``"active-consumers"`` trims values once every +attached consumer has read them; consumers created later start at the current +trim watermark instead of position 0. +""" + +# Compact the backing list only once the consumed prefix is at least this long +# (and at least half the list), so trimming stays amortized O(1) per value. +BUFFER_COMPACTION_MIN_HEAD = 1024 + +# Marks a consumed slot that has been released. A dedicated sentinel (rather +# than upstream's `undefined`) because `None` is a legitimate stream value. +_CLEARED: Any = object() + + +class _ConsumerState: + __slots__ = ("position", "waiter", "cancelled") + + def __init__(self, position: int) -> None: + self.position = position + self.waiter: Optional[asyncio.Future[None]] = None + self.cancelled = False + + +class _ReplayBuffer(Generic[T]): + """Buffer + consumer bookkeeping shared by the stream and the broadcaster. + + Consumer positions are absolute. Buffer index is + ``buffer_head + position - trim_offset``. + """ + + _invariant_message = "replay buffer invariant violated: consumed slot was cleared" + + def __init__(self, stream_replay: StreamReplay = "full") -> None: + self._stream_replay: StreamReplay = stream_replay self._buffer: List[Any] = [] + self._buffer_head = 0 + self._trim_offset = 0 + self._consumers: Dict[int, _ConsumerState] = {} + self._next_consumer_id = 0 + + @property + def stream_replay(self) -> StreamReplay: + return self._stream_replay + + # -- hooks the subclasses implement ------------------------------------ + + def _on_consumer_attached(self) -> None: + """Called after a consumer is registered (e.g. to start a pump).""" + + def _terminal_state(self, consumer_id: int) -> bool: + """Return True (consumer done) or raise, when the consumer is caught up + and the source has finished. Return False to wait for more data.""" + raise NotImplementedError + + # -- buffer management ------------------------------------------------- + + def _append(self, value: T) -> None: + self._buffer.append(value) + + def _buffer_index(self, state: _ConsumerState) -> int: + return self._buffer_head + state.position - self._trim_offset + + def _has_data(self, state: _ConsumerState) -> bool: + return self._buffer_index(state) < len(self._buffer) + + def find_last_buffered(self, predicate: Callable[[T], bool]) -> Optional[T]: + """Synchronously scan the retained buffer from the end, returning the + last item matching ``predicate``. + + Sees only what has been buffered (and retained) so far. Unlike a + consumer replay this costs no per-item await. + """ + for index in range(len(self._buffer) - 1, self._buffer_head - 1, -1): + item = self._buffer[index] + if item is not _CLEARED and predicate(item): + return item + return None + + def _trim_consumed(self) -> None: + if self._stream_replay == "full": + return + if not self._consumers: + self._drop_unread_backlog() + return + + minimum = min(state.position for state in self._consumers.values()) + next_head = self._buffer_head + minimum - self._trim_offset + if next_head <= self._buffer_head: + return + + self._trim_offset = minimum + if next_head == len(self._buffer): + self._buffer = [] + self._buffer_head = 0 + return + + for index in range(self._buffer_head, next_head): + self._buffer[index] = _CLEARED + if next_head >= BUFFER_COMPACTION_MIN_HEAD and next_head * 2 >= len(self._buffer): + self._buffer = self._buffer[next_head:] + self._buffer_head = 0 + return + self._buffer_head = next_head + + def _drop_unread_backlog(self) -> None: + """Drop the retained backlog once no consumer can ever read it again. + + Before the first consumer joins, the backlog IS the catch-up history + late joiners replay, so it is retained. Once at least one consumer has + existed and none remain, any future consumer starts at the watermark + anyway, so the retained values would stay pinned for nothing. + """ + if self._next_consumer_id == 0: + return + dropped = len(self._buffer) - self._buffer_head + if dropped == 0: + return + self._trim_offset += dropped + self._buffer = [] + self._buffer_head = 0 + + def _notify_waiting(self) -> None: + for state in self._consumers.values(): + waiter = state.waiter + state.waiter = None + if waiter is not None and not waiter.done(): + waiter.set_result(None) + + # -- consumers --------------------------------------------------------- + + def create_consumer(self) -> "ReplayConsumer[T]": + consumer_id = self._next_consumer_id + self._next_consumer_id += 1 + self._consumers[consumer_id] = _ConsumerState(self._trim_offset) + self._on_consumer_attached() + return ReplayConsumer(self, consumer_id) + + def _detach(self, consumer_id: int, error: Optional[BaseException] = None) -> None: + state = self._consumers.get(consumer_id) + if state is None: + return + state.cancelled = True + # Settle a pending __anext__ so it does not hang forever on a consumer + # that has just been removed: it either re-runs and reports done, or + # raises the same error passed to athrow(). + waiter = state.waiter + state.waiter = None + if waiter is not None and not waiter.done(): + if error is None: + waiter.set_result(None) + else: + waiter.set_exception(error) + del self._consumers[consumer_id] + self._trim_consumed() + self._after_detach() + + def _after_detach(self) -> None: + """Called after a consumer is removed via aclose()/athrow().""" + + async def _next(self, consumer_id: int) -> T: + while True: + state = self._consumers.get(consumer_id) + if state is None or state.cancelled: + raise StopAsyncIteration + + index = self._buffer_index(state) + if index < len(self._buffer): + value = self._buffer[index] + if value is _CLEARED: + raise RuntimeError(f"{type(self).__name__} buffer invariant violated: consumed slot was cleared") + state.position += 1 + self._trim_consumed() + return value + + if self._terminal_state(consumer_id): + raise StopAsyncIteration + + waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future() + state.waiter = waiter + try: + await waiter + finally: + if state.waiter is waiter: + state.waiter = None + + +class ReplayConsumer(AsyncIterator[T]): + """Independent read cursor over a replay buffer. + + Unlike a plain async generator, ``aclose()`` / ``athrow()`` may be called + while another task is awaiting ``__anext__``: the pending read is settled + (done, or the thrown error) instead of hanging. + """ + + def __init__(self, owner: _ReplayBuffer[T], consumer_id: int) -> None: + self._owner = owner + self._consumer_id = consumer_id + + def __aiter__(self) -> "ReplayConsumer[T]": + return self + + async def __anext__(self) -> T: + return await self._owner._next(self._consumer_id) + + async def aclose(self) -> None: + self._owner._detach(self._consumer_id) + + async def athrow(self, error: Union[BaseException, type]) -> T: + exc: BaseException + if isinstance(error, BaseException): + exc = error + elif isinstance(error, type) and issubclass(error, BaseException): + exc = error() + else: + exc = Exception(str(error)) + self._owner._detach(self._consumer_id, exc) + raise exc + + +class ReusableReadableStream(_ReplayBuffer[T]): + """Buffer a single async source and fan it out to many consumers. + + Options (keyword-only, mirroring upstream ``ReusableReadableStreamOptions``): + + - ``stream_replay``: ``"full"`` (default) or ``"active-consumers"``. + - ``on_value``: called synchronously with each value as it is buffered. + - ``is_terminal_value``: when it returns True for a value, that value is the + final buffered event; the source is closed once and pumping stops. + """ + + def __init__( + self, + source: AsyncIterable[T], + *, + stream_replay: StreamReplay = "full", + on_value: Optional[Callable[[T], None]] = None, + is_terminal_value: Optional[Callable[[T], bool]] = None, + ) -> None: + super().__init__(stream_replay) + self._source = source + self._source_iterator: Optional[AsyncIterator[T]] = None self._complete = False self._error: Optional[BaseException] = None self._started = False - self._condition = asyncio.Condition() - - def _ensure_started(self) -> None: - if not self._started: - self._started = True - asyncio.create_task(self._pump()) + self._pump_task: Optional[asyncio.Task[None]] = None + self._source_closed = False + self._on_value = on_value + self._is_terminal_value = is_terminal_value @property def is_complete(self) -> bool: @@ -25,35 +291,91 @@ def is_complete(self) -> bool: buffer without waiting on the source.""" return self._complete + def _on_consumer_attached(self) -> None: + if not self._started: + self._started = True + self._pump_task = asyncio.get_running_loop().create_task(self._pump()) + + def _terminal_state(self, consumer_id: int) -> bool: + if self._error is not None: + self._consumers.pop(consumer_id, None) + raise self._error + if self._complete: + self._consumers.pop(consumer_id, None) + return True + return False + + async def _close_source(self) -> None: + """Close the source iterator at most once (terminal value / cancel).""" + if self._source_closed: + return + self._source_closed = True + iterator = self._source_iterator + aclose = getattr(iterator, "aclose", None) + if aclose is not None: + await aclose() + async def _pump(self) -> None: + iterator = self._source.__aiter__() + self._source_iterator = iterator try: - async for item in self._source: - async with self._condition: - self._buffer.append(item) - self._condition.notify_all() - except BaseException as exc: + while True: + try: + value = await iterator.__anext__() + except StopAsyncIteration: + break + self._append(value) + if self._on_value is not None: + self._on_value(value) + self._notify_waiting() + if self._is_terminal_value is not None and self._is_terminal_value(value): + self._complete = True + self._notify_waiting() + try: + await self._close_source() + except Exception: + # The terminal event is authoritative; closing is cleanup only. + pass + break + except asyncio.CancelledError: + # cancel() tore the pump down; consumers observe completion. + pass + except Exception as exc: self._error = exc finally: - async with self._condition: - self._complete = True - self._condition.notify_all() + self._complete = True + self._notify_waiting() + + async def cancel(self) -> None: + """Cancel the source and every consumer, dropping the retained backlog.""" + for state in self._consumers.values(): + state.cancelled = True + waiter = state.waiter + state.waiter = None + if waiter is not None and not waiter.done(): + waiter.set_result(None) + self._consumers.clear() + # Cancellation is terminal: nobody can read the backlog anymore. + self._drop_unread_backlog() + task = self._pump_task + if task is not None and not task.done() and task is not asyncio.current_task(): + task.cancel() + # The pump swallows its own cancellation, so awaiting it only + # raises if the caller of cancel() is itself cancelled. + await task + self._complete = True + try: + await self._close_source() + except Exception: + pass + # The pump may have landed one in-flight value before it was + # cancelled; sweep once more so the buffer is empty regardless. + self._drop_unread_backlog() - def create_consumer(self) -> AsyncIterator[Any]: - self._ensure_started() - async def gen() -> AsyncIterator[Any]: - index = 0 - while True: - async with self._condition: - while index >= len(self._buffer) and not self._complete: - await self._condition.wait() - if index < len(self._buffer): - item = self._buffer[index] - index += 1 - elif self._error is not None: - raise self._error - else: - break - yield item - - return gen() +__all__ = [ + "BUFFER_COMPACTION_MIN_HEAD", + "ReplayConsumer", + "ReusableReadableStream", + "StreamReplay", +] diff --git a/src/openrouter_agent/stop_conditions.py b/src/openrouter_agent/stop_conditions.py index 6db0245..9e1c34f 100644 --- a/src/openrouter_agent/stop_conditions.py +++ b/src/openrouter_agent/stop_conditions.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio from typing import Any, Callable, Mapping, Sequence from ._utils import get_field, maybe_await @@ -18,7 +19,7 @@ def check(options: Mapping[str, Any]) -> bool: return any( getattr(call, "name", None) == name or (isinstance(call, Mapping) and call.get("name") == name) for step in _steps(options) - for call in get_field(step, "tool_calls", []) + for call in (get_field(step, "tool_calls", None) or get_field(step, "toolCalls", None) or []) ) return check @@ -59,7 +60,12 @@ def finish_reason_is(reason: str) -> Callable[[Mapping[str, Any]], bool]: async def is_stop_condition_met( stop_conditions: Sequence[Callable[[Mapping[str, Any]], Any]], steps: Sequence[Any] ) -> bool: - for condition in stop_conditions: - if await maybe_await(condition({"steps": steps})): - return True - return False + """Evaluate every stop condition (OR logic). + + Matches upstream `isStopConditionMet`: all conditions are evaluated + concurrently (no short-circuit), a failing condition propagates its error, + and only a result that is exactly ``True`` stops — ``None`` or other truthy + values do not. + """ + results = await asyncio.gather(*(maybe_await(condition({"steps": steps})) for condition in stop_conditions)) + return any(result is True for result in results) diff --git a/src/openrouter_agent/stream_transformers.py b/src/openrouter_agent/stream_transformers.py index 54c7895..bba2d62 100644 --- a/src/openrouter_agent/stream_transformers.py +++ b/src/openrouter_agent/stream_transformers.py @@ -38,8 +38,41 @@ def extract_responses_message_from_response(response: Any) -> Any: raise ValueError("Response does not contain a message output item") +def is_truncated_at_max_output_tokens(response: Any) -> bool: + """Whether the provider stopped this response at `max_output_tokens`. + + The `function_call` items on such a response are an unfinished batch: the + model asked for the set together, and the last one carries whatever + argument prefix fit in the budget. Executing the complete ones would hand + the model a result set with a silent hole, and spend side effects on a turn + that truncates the same way on the same budget, so none of them run. The + caller sees every item and `incomplete_details`, and resumes once the + budget is raised. + """ + if get_field(response, "status") != "incomplete": + return False + details = get_field(response, "incompleteDetails", get_field(response, "incomplete_details")) + return get_field(details, "reason") == "max_output_tokens" if details is not None else False + + +def response_has_tool_calls(response: Any) -> bool: + """Whether a response carries tool calls the loop should execute. A + response truncated at `max_output_tokens` has none, even when its output + carries a cut-off `function_call` item.""" + if is_truncated_at_max_output_tokens(response): + return False + return any(get_field(item, "type") == "function_call" for item in _output_items(response)) + + def extract_tool_calls_from_response(response: Any) -> List[ParsedToolCall]: + """Extract all tool calls from a completed response. + + A response truncated at `max_output_tokens` yields no tool calls: see + `is_truncated_at_max_output_tokens`. + """ calls: List[ParsedToolCall] = [] + if is_truncated_at_max_output_tokens(response): + return calls for item in _output_items(response): if get_field(item, "type") != "function_call": continue diff --git a/src/openrouter_agent/tool.py b/src/openrouter_agent/tool.py index 4bd894b..e866736 100644 --- a/src/openrouter_agent/tool.py +++ b/src/openrouter_agent/tool.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, Optional +from typing import Any, Dict, Mapping, Optional from .tool_types import SHARED_CONTEXT_KEY, ToolType @@ -19,7 +19,20 @@ def tool( on_tool_called: Any = None, on_response_received: Any = None, to_model_output: Any = None, + strict: Optional[bool] = None, + wire_input_schema: Optional[Mapping[str, Any]] = None, + loop_key: Any = None, ) -> Dict[str, Any]: + """Create a client tool. + + `strict` is passed through to the wire function-tool definition so + providers can enforce structured-outputs-style argument adherence. + `wire_input_schema` is a caller-owned JSON Schema serialized to the model + verbatim (sanitized) in place of the one generated from `input_schema`, + which keeps validating the call. `loop_key` declares the call identity + used by doom-loop detection: a callable over the validated arguments + (returning `None` exempts the call) or `False` (statically exempt). + """ if name == SHARED_CONTEXT_KEY: raise ValueError('Tool name "shared" is reserved for shared context. Choose a different name.') fn: Dict[str, Any] = {"name": name, "input_schema": input_schema} @@ -37,6 +50,12 @@ def tool( fn["require_approval"] = require_approval if to_model_output is not None: fn["to_model_output"] = to_model_output + if strict is not None: + fn["strict"] = strict + if wire_input_schema is not None: + fn["wire_input_schema"] = wire_input_schema + if loop_key is not None: + fn["loop_key"] = loop_key if on_tool_called is not None: if output_schema is None: raise ValueError(f'HITL tool "{name}" must declare an output_schema.') @@ -50,15 +69,36 @@ def tool( return {"type": ToolType.Function.value, "function": fn} -def server_tool(config: Dict[str, Any]) -> Dict[str, Any]: - return {"_brand": "server-tool", "config": dict(config)} +def server_tool(config: Mapping[str, Any], options: Optional[Mapping[str, Any]] = None) -> Dict[str, Any]: + """Create an OpenRouter server-executed tool. + + Each server tool carries a stable tool-set `id` (default + `server:{config["type"]}`) so activation APIs (`create_tool_set`) can + address it. Override it with `options={"id": ...}`. + """ + explicit_id = (options or {}).get("id") + if explicit_id == "": + raise ValueError("Server tool ID must not be empty") + tool_id = explicit_id if explicit_id is not None else f"server:{config.get('type')}" + return {"_brand": "server-tool", "config": dict(config), "id": tool_id} -def mark_mcp(tool_to_mark: Dict[str, Any]) -> Dict[str, Any]: +def mark_mcp(tool_to_mark: Dict[str, Any], options: Optional[Mapping[str, Any]] = None) -> Dict[str, Any]: """Add the additive MCP brand to an already-built client tool (see `is_mcp_tool`). Non-mutating: returns a shallow copy carrying `_mcp`, so the tool's runtime behavior and wire shape are unchanged -- only the `is_mcp_tool` check now identifies it as MCP-originated. Used by `@openrouter/mcp`-equivalent integrations to mark wrapped remote tools. """ - return {**tool_to_mark, "_mcp": True} + marked = {**tool_to_mark, "_mcp": True} + # `options["loop_key"]` attaches a doom-loop identity to the wrapped tool; + # `options["wire_input_schema"]` is the server's own JSON Schema, sent to + # the model in place of one regenerated from `input_schema`. + if options and isinstance(marked.get("function"), Mapping): + fn = dict(marked["function"]) + if options.get("loop_key") is not None: + fn["loop_key"] = options["loop_key"] + if options.get("wire_input_schema") is not None: + fn["wire_input_schema"] = options["wire_input_schema"] + marked["function"] = fn + return marked diff --git a/src/openrouter_agent/tool_event_broadcaster.py b/src/openrouter_agent/tool_event_broadcaster.py index 05f1bb5..6770720 100644 --- a/src/openrouter_agent/tool_event_broadcaster.py +++ b/src/openrouter_agent/tool_event_broadcaster.py @@ -1,62 +1,82 @@ +"""Push-based multi-consumer event broadcaster. + +Ports `packages/agent/src/lib/tool-event-broadcaster.ts`. Similar to +`ReusableReadableStream` but for push-based events from tool execution. + +Each consumer gets its own position in the buffer. Full replay is the default; +``stream_replay="active-consumers"`` compacts events once every attached +consumer has read them. + +Contract: ``push()`` after ``complete()`` is a silent no-op, so callers must +finish pushing before completing (see tests/unit/test_turn_end_race_condition.py). +""" + from __future__ import annotations import asyncio -from typing import Any, AsyncIterator, List, Optional +from typing import Optional, TypeVar + +from .reusable_stream import ReplayConsumer, StreamReplay, _ReplayBuffer -_SENTINEL = object() +T = TypeVar("T") -class ToolEventBroadcaster: - def __init__(self) -> None: - self._buffer: List[Any] = [] +class ToolEventBroadcaster(_ReplayBuffer[T]): + def __init__(self, stream_replay: StreamReplay = "full") -> None: + super().__init__(stream_replay) self._complete = False self._error: Optional[BaseException] = None - self._condition = asyncio.Condition() - def push(self, event: Any) -> None: + @property + def is_complete(self) -> bool: + return self._complete + + def push(self, event: T) -> None: + """Push an event to all consumers. Ignored once complete.""" if self._complete: return - self._buffer.append(event) - self._wake() + self._append(event) + self._notify_waiting() def complete(self, error: Optional[BaseException] = None) -> None: + """Mark the broadcaster complete, optionally signalling ``error`` to every + consumer after it drains the buffered events. Idempotent.""" if self._complete: return self._complete = True self._error = error - self._wake() - - def error(self, error: BaseException) -> None: - self.complete(error) - - def _wake(self) -> None: + self._notify_waiting() + # Upstream schedules cleanup on a microtask so consumers created in + # the same synchronous section still see the buffer. try: loop = asyncio.get_running_loop() except RuntimeError: return - loop.create_task(self._notify()) - - async def _notify(self) -> None: - async with self._condition: - self._condition.notify_all() - - def create_consumer(self) -> AsyncIterator[Any]: - async def gen() -> AsyncIterator[Any]: - index = 0 - while True: - async with self._condition: - while index >= len(self._buffer) and not self._complete: - await self._condition.wait() - if index < len(self._buffer): - item = self._buffer[index] - index += 1 - elif self._error is not None: - raise self._error - else: - break - yield item - - return gen() - - def __aiter__(self) -> AsyncIterator[Any]: + loop.call_soon(self._cleanup) + + def error(self, error: BaseException) -> None: + self.complete(error) + + def _cleanup(self) -> None: + # Only release when complete and every consumer is done. + if self._stream_replay == "active-consumers" and self._complete and not self._consumers: + self._buffer = [] + self._buffer_head = 0 + + def _after_detach(self) -> None: + self._cleanup() + + def _terminal_state(self, consumer_id: int) -> bool: + if not self._complete: + return False + self._consumers.pop(consumer_id, None) + self._cleanup() + if self._error is not None: + raise self._error + return True + + def __aiter__(self) -> ReplayConsumer[T]: return self.create_consumer() + + +__all__ = ["ToolEventBroadcaster"] diff --git a/src/openrouter_agent/tool_executor.py b/src/openrouter_agent/tool_executor.py index a768529..a7f9d26 100644 --- a/src/openrouter_agent/tool_executor.py +++ b/src/openrouter_agent/tool_executor.py @@ -44,13 +44,18 @@ def convert_tools_to_api_format(tools: Sequence[Tool]) -> List[Dict[str, Any]]: converted.append(dict(config) if isinstance(config, Mapping) else {}) continue fn = get_tool_function(item) + wire_schema = fn.get("wire_input_schema") api: Dict[str, Any] = { "type": "function", "name": fn.get("name"), - "parameters": schema_to_json_schema(fn.get("input_schema")), + "parameters": sanitize_json_schema(dict(wire_schema)) + if wire_schema is not None + else schema_to_json_schema(fn.get("input_schema")), } if fn.get("description") is not None: api["description"] = fn["description"] + if fn.get("strict") is not None: + api["strict"] = fn["strict"] converted.append(api) return converted @@ -62,7 +67,7 @@ async def execute_regular_tool( source = "mcp" if is_mcp_tool(tool) else "client" try: args = validate_schema(fn.get("input_schema"), tool_call.arguments) - result = await maybe_await(fn["execute"](args, context)) + result = await maybe_await(fn["execute"](args, _with_tool_call(context, tool_call))) if fn.get("output_schema") is not None: result = validate_schema(fn.get("output_schema"), result) return {"tool_call_id": tool_call.id, "tool_name": tool_call.name, "source": source, "result": result} @@ -84,10 +89,13 @@ async def execute_generator_tool( ) -> Dict[str, Any]: fn = get_tool_function(tool) source = "mcp" if is_mcp_tool(tool) else "client" - preliminary: List[Any] = [] + # Upstream (DEV-817) stopped accumulating every yield into a per-call + # `preliminary_results` array: yields are broadcast live through + # `on_preliminary_result` only, and the terminal result no longer carries + # the yield history. try: args = validate_schema(fn.get("input_schema"), tool_call.arguments) - produced = fn["execute"](args, context) + produced = fn["execute"](args, _with_tool_call(context, tool_call)) if not hasattr(produced, "__aiter__"): produced = await maybe_await(produced) final = None @@ -99,7 +107,6 @@ async def execute_generator_tool( async for value in produced: if broad_overlapping_schemas: if pending_broad_event is not None: - preliminary.append(pending_broad_event) if on_preliminary_result is not None: await maybe_await(on_preliminary_result(tool_call.id, pending_broad_event)) pending_broad_event = value @@ -116,7 +123,6 @@ async def execute_generator_tool( continue if fn.get("event_schema") is not None: value = validate_schema(fn.get("event_schema"), value) - preliminary.append(value) if on_preliminary_result is not None: await maybe_await(on_preliminary_result(tool_call.id, value)) if broad_overlapping_schemas and pending_broad_event is not None: @@ -131,7 +137,6 @@ async def execute_generator_tool( "tool_name": tool_call.name, "source": source, "result": final, - "preliminary_results": preliminary, } except Exception as exc: return { @@ -139,7 +144,6 @@ async def execute_generator_tool( "tool_name": tool_call.name, "source": source, "result": None, - "preliminary_results": preliminary, "error": exc, } @@ -151,7 +155,7 @@ async def execute_hitl_tool( source = "mcp" if is_mcp_tool(tool) else "client" try: args = validate_schema(fn.get("input_schema"), tool_call.arguments) - result = await maybe_await(fn["on_tool_called"](args, context)) + result = await maybe_await(fn["on_tool_called"](args, _with_tool_call(context, tool_call))) if result is None: return None result = validate_schema(fn.get("output_schema"), result) @@ -166,6 +170,32 @@ async def execute_hitl_tool( } +def _with_tool_call(context: Optional[Mapping[str, Any]], tool_call: ParsedToolCall) -> Dict[str, Any]: + """Thread the executed call into the execute context. `tool_call` is part + of the tool-facing contract: a caller-provided one wins; otherwise the + executed call fills the gap as a wire-shaped function_call item. Every + executor entry point applies this (upstream `buildExecuteCtx`).""" + merged = dict(context or {}) + if not merged.get("tool_call"): + merged["tool_call"] = to_function_call_item(tool_call) + return merged + + +def to_function_call_item(tool_call: ParsedToolCall) -> Dict[str, Any]: + """Convert an executor-shaped `ParsedToolCall` back into the wire-shaped + function_call item the execute context declares. Arguments are always + re-serialized (semantically equal to the wire arguments, not guaranteed + byte-identical).""" + arguments = tool_call.arguments + return { + "type": "function_call", + "id": tool_call.id, + "callId": tool_call.id, + "name": tool_call.name, + "arguments": json_dumps(arguments) if arguments is not None else "{}", + } + + async def execute_tool( tool: Tool, tool_call: ParsedToolCall, diff --git a/src/openrouter_agent/tool_set.py b/src/openrouter_agent/tool_set.py new file mode 100644 index 0000000..92b1d0e --- /dev/null +++ b/src/openrouter_agent/tool_set.py @@ -0,0 +1,581 @@ +"""Declarative, state- and context-aware tool activation. + +Port of upstream ``packages/agent/src/lib/tool-set.ts`` and +``packages/agent/src/lib/tool-set-types.ts`` (``@openrouter/agent/tool-set``). + +Third-party notice (mirrors upstream ``packages/agent/THIRD_PARTY_NOTICES.md``): +``@openrouter/agent/tool-set`` -- and therefore this module -- is adapted from +``ai-tool-set`` v1.0.0 (https://github.com/zirkelc/ai-tool-set/tree/v1.0.0), +which is licensed under the MIT License: + + MIT License + + Copyright (c) 2024 Chris + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +Usage:: + + ts = create_tool_set(tools=[search, write, server_tool({"type": "openrouter:datetime"})]) + ts = ts.deactivate("write").activate_when("write", lambda i: (i.get("context") or {}).get("admin") is True) + resolved = ts.resolve({"context": {"admin": True}}) + result = call_model(client, {"model": "...", "input": "...", **resolved.call_model}) + +Divergences from upstream (see the contract's Idiomatic Divergences): + +- The TypeScript three-way compile-time partition (``Partition``, + ``FilterToolsByIds``, ``Infer*Ids``, ``WidenedPartition`` ...) is type-level + only and is not reproduced (divergence 4). Runtime behavior -- including + mutable-vs-immutable aliasing, last-call-wins, situation overlays and the + exhaustive ``status_by_tool`` snapshot -- is ported exactly. +- ``resolve()`` / ``resolve_situation()`` return a frozen + :class:`ResolvedToolSnapshot` dataclass. Its ``call_model`` attribute is the + spread-safe ``{"tools": ..., "active_tools": ...}`` mapping. ``infer_tools()`` + returns a plain ``dict`` carrying the :data:`TOOL_SET_SNAPSHOT` sentinel key + (the Python analogue of upstream's ``Symbol.for(...)`` marker, which survives + ``{**snapshot}``), so :func:`strip_tool_set_snapshot_metadata` can remove its + metadata when the whole dict is spread into a ``call_model`` request. +- String literals that are *data* (status ``reason``, ``directive``, situation + rule ``mode``: ``"activateWhen"`` / ``"deactivateWhen"``) keep upstream's + values verbatim, like event type strings elsewhere in this port. +- Upstream ``serverTool(config, { id })`` stamps a stable ``id`` on every server + tool. This port's ``server_tool`` has no ``id`` option yet, so + :func:`get_server_tool_id` applies the same rule: an explicit non-empty + ``id`` key on the tool wins, else ``f"server:{config['type']}"``. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Literal, Mapping, Optional, Sequence, Tuple, Union + +from typing_extensions import TypeAlias, TypedDict + +from .tool_types import ConversationState, get_tool_function, is_server_tool + + +class _ToolSetSnapshotMarker: + """Sentinel type for :data:`TOOL_SET_SNAPSHOT` (upstream ``Symbol.for``).""" + + _instance: Optional[_ToolSetSnapshotMarker] = None + + def __new__(cls) -> _ToolSetSnapshotMarker: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __repr__(self) -> str: + return "TOOL_SET_SNAPSHOT" + + def __reduce__(self) -> str: + return "TOOL_SET_SNAPSHOT" + + +#: Identifies dicts produced by the tool-set module (upstream ``TOOL_SET_SNAPSHOT`` +#: in ``async-params.ts``). A non-string dict key, so it survives ``{**snapshot}`` +#: but can never collide with a request field name. +TOOL_SET_SNAPSHOT: Any = _ToolSetSnapshotMarker() + +_TOOL_SET_SNAPSHOT_METADATA_KEYS = ("enabled", "disabled", "status_by_tool", "call_model") + + +def strip_tool_set_snapshot_metadata(request: Dict[Any, Any]) -> None: + """Remove tool-set metadata only from marked snapshots or their spreads. + + Port of upstream ``stripToolSetSnapshotMetadata`` (``async-params.ts``). + Mutates ``request`` in place. Unmarked requests are left untouched, so + identically named user fields survive. + """ + if request.get(TOOL_SET_SNAPSHOT) is not True: + return + for key in _TOOL_SET_SNAPSHOT_METADATA_KEYS: + request.pop(key, None) + request.pop(TOOL_SET_SNAPSHOT, None) + + +# ─── public types ──────────────────────────────────────────────────────────── + + +class ActivationInput(TypedDict, total=False): + """Input handed to every activation predicate.""" + + state: ConversationState + context: Mapping[str, Any] + + +ActivationPredicate: TypeAlias = Callable[[ActivationInput], Any] + +ConditionalMode = Literal["activateWhen", "deactivateWhen"] + +StatusReason = Literal["default", "activate", "deactivate", "activateWhen", "deactivateWhen", "situation"] + +Directive = Literal["activate", "deactivate", "activateWhen", "deactivateWhen"] + + +class SituationConditionalRuleObject(TypedDict, total=False): + mode: ConditionalMode + predicate: ActivationPredicate + + +SituationConditionalRule: TypeAlias = Union[ActivationPredicate, SituationConditionalRuleObject] + + +class SituationConfig(TypedDict, total=False): + """Declarative fixed partition overlay for one named situation. + + Ids not listed keep the base ToolSet partition after the overlay. + ``conditional`` rules default to ``activateWhen`` mode; use + ``{"mode": "deactivateWhen", "predicate": fn}`` for the reverse default. + """ + + enabled: Sequence[str] + disabled: Sequence[str] + conditional: Mapping[str, Optional[SituationConditionalRule]] + + +class ToolStatusEntry(TypedDict, total=False): + #: Whether the tool resolved active. + enabled: bool + reason: StatusReason + #: The last applicable directive before predicates ran. Absent when the + #: tool is still at its construction default. + directive: Directive + #: True when the final state depended on evaluating a runtime predicate. + predicate: bool + + +@dataclass(frozen=True) +class ResolvedToolSnapshot: + """What :meth:`ToolSet.resolve` / :meth:`ToolSet.resolve_situation` return.""" + + #: Active tools only, construction order preserved. + tools: List[Any] + #: Active client tool names only (``call_model`` ``active_tools`` wire + #: format). Server tool ids are omitted. + active_tools: List[str] + #: Spread-safe ``call_model`` input: exactly ``{"tools", "active_tools"}``. + call_model: Dict[str, Any] + #: Ids that resolved active (client + server). + enabled: List[str] + #: Ids that resolved inactive (client + server). + disabled: List[str] + #: Exhaustive id -> status entry; every known id is present. + status_by_tool: Dict[str, ToolStatusEntry] = field(default_factory=dict) + + +# ─── internals ─────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class _ActivationEntry: + kind: Literal["static", "activateWhen", "deactivateWhen"] + source: Literal["default", "activate", "deactivate", "activateWhen", "deactivateWhen", "situation"] + active: bool = False + predicate: Optional[ActivationPredicate] = None + + +@dataclass(frozen=True) +class _SituationRuntime: + enabled: Tuple[str, ...] + disabled: Tuple[str, ...] + conditional: Tuple[Tuple[str, ConditionalMode, ActivationPredicate], ...] + + +@dataclass(frozen=True) +class _IndexedTools: + ordered_tools: Tuple[Any, ...] + ordered_ids: Tuple[str, ...] + tool_by_id: Dict[str, Any] + client_names: frozenset[str] + server_ids: frozenset[str] + + +def get_server_tool_id(tool: Mapping[str, Any]) -> str: + """Stable tool-set id of a server tool (upstream ``defaultServerId``). + + An explicit non-empty string ``id`` wins; otherwise ``server:{config.type}``. + The prefix guarantees it can never collide with a client function name. + """ + explicit = tool.get("id") + if isinstance(explicit, str) and len(explicit) > 0: + return explicit + config = tool.get("config") + tool_type = config.get("type") if isinstance(config, Mapping) else None + return f"server:{tool_type}" + + +def get_tool_id(tool: Mapping[str, Any]) -> str: + """Addressable tool-set id: server id for server tools, else ``function.name``.""" + if is_server_tool(tool): + return get_server_tool_id(tool) + return str(get_tool_function(tool).get("name")) + + +def _index_tools(tools: Sequence[Any]) -> _IndexedTools: + tool_by_id: Dict[str, Any] = {} + ordered_ids: List[str] = [] + client_names: set[str] = set() + server_ids: set[str] = set() + for t in tools: + tid = get_tool_id(t) + if tid in tool_by_id: + raise ValueError(f'Duplicate tool ID: "{tid}"') + tool_by_id[tid] = t + ordered_ids.append(tid) + if is_server_tool(t): + server_ids.add(tid) + else: + client_names.add(tid) + return _IndexedTools( + ordered_tools=tuple(tools), + ordered_ids=tuple(ordered_ids), + tool_by_id=tool_by_id, + client_names=frozenset(client_names), + server_ids=frozenset(server_ids), + ) + + +def _to_id_list(names: Union[str, Sequence[str]]) -> List[str]: + return [names] if isinstance(names, str) else list(names) + + +def _normalize_conditional_rule(situation: str, tool_id: str, rule: Any) -> Tuple[ConditionalMode, ActivationPredicate]: + if callable(rule): + return "activateWhen", rule + mode = rule.get("mode") if isinstance(rule, Mapping) else None + if ( + not isinstance(rule, Mapping) + or not callable(rule.get("predicate")) + or (mode is not None and mode not in ("activateWhen", "deactivateWhen")) + ): + raise ValueError( + f'Situation "{situation}": conditional rule for tool "{tool_id}" ' + "must be a function or { mode, predicate } object" + ) + return (mode or "activateWhen"), rule["predicate"] + + +class ToolSet: + """Immutable-by-default stateful set of tools. + + Each tool id is enabled (default / :meth:`activate`), disabled + (:meth:`deactivate`) or conditional (:meth:`activate_when` / + :meth:`deactivate_when`); the most recent directive per id wins. Named + situations (:meth:`define_situations`) overlay that base partition. + + Immutable instances return a new ``ToolSet`` from every mutator; mutable + instances (``mutable=True``) mutate in place and return ``self``. + """ + + __slots__ = ("_index", "_activation", "_situations", "_mutable") + + def __init__( + self, + index: _IndexedTools, + activation: Dict[str, _ActivationEntry], + situations: Dict[str, _SituationRuntime], + mutable: bool, + ) -> None: + self._index = index + self._activation = activation + self._situations = situations + self._mutable = mutable + + @classmethod + def create(cls, *, tools: Sequence[Any], mutable: bool = False) -> ToolSet: + """Internal factory. Prefer :func:`create_tool_set`.""" + return cls(_index_tools(tools), {}, {}, bool(mutable)) + + @property + def tools(self) -> Tuple[Any, ...]: + """All tools in construction order, regardless of activation state.""" + return self._index.ordered_tools + + @property + def mutable(self) -> bool: + return self._mutable + + def _assert_known(self, tool_id: str) -> None: + if tool_id not in self._index.tool_by_id: + raise ValueError(f'Unknown tool: "{tool_id}"') + + def _with_partition_mutation(self, mutate: Callable[[Dict[str, _ActivationEntry]], None]) -> ToolSet: + if self._mutable: + mutate(self._activation) + return self + next_activation = dict(self._activation) + mutate(next_activation) + return ToolSet(self._index, next_activation, self._situations, False) + + def activate(self, names: Union[str, Sequence[str]]) -> ToolSet: + ids = _to_id_list(names) + for n in ids: + self._assert_known(n) + + def mutate(activation: Dict[str, _ActivationEntry]) -> None: + for n in ids: + activation[n] = _ActivationEntry(kind="static", active=True, source="activate") + + return self._with_partition_mutation(mutate) + + def deactivate(self, names: Union[str, Sequence[str]]) -> ToolSet: + ids = _to_id_list(names) + for n in ids: + self._assert_known(n) + + def mutate(activation: Dict[str, _ActivationEntry]) -> None: + for n in ids: + activation[n] = _ActivationEntry(kind="static", active=False, source="deactivate") + + return self._with_partition_mutation(mutate) + + def activate_when( + self, + name_or_map: Union[str, Mapping[str, Optional[ActivationPredicate]]], + predicate: Optional[ActivationPredicate] = None, + ) -> ToolSet: + """Inactive unless ``predicate(input) is True`` at resolve time. + + Accepts ``(name, predicate)`` or a ``{name: predicate}`` map. + """ + entries = self._normalize_predicate_arg(name_or_map, predicate) + + def mutate(activation: Dict[str, _ActivationEntry]) -> None: + for n, p in entries: + activation[n] = _ActivationEntry(kind="activateWhen", predicate=p, source="activateWhen") + + return self._with_partition_mutation(mutate) + + def deactivate_when( + self, + name_or_map: Union[str, Mapping[str, Optional[ActivationPredicate]]], + predicate: Optional[ActivationPredicate] = None, + ) -> ToolSet: + """Active unless ``predicate(input) is True`` at resolve time. + + Accepts ``(name, predicate)`` or a ``{name: predicate}`` map. + """ + entries = self._normalize_predicate_arg(name_or_map, predicate) + + def mutate(activation: Dict[str, _ActivationEntry]) -> None: + for n, p in entries: + activation[n] = _ActivationEntry(kind="deactivateWhen", predicate=p, source="deactivateWhen") + + return self._with_partition_mutation(mutate) + + def _normalize_predicate_arg( + self, + name_or_map: Union[str, Mapping[str, Optional[ActivationPredicate]]], + predicate: Optional[ActivationPredicate], + ) -> List[Tuple[str, ActivationPredicate]]: + if isinstance(name_or_map, str): + if not predicate: + raise ValueError("activate_when/deactivate_when requires a predicate when called with a name") + self._assert_known(name_or_map) + return [(name_or_map, predicate)] + if not isinstance(name_or_map, Mapping): + raise ValueError("activate_when/deactivate_when requires a name+predicate or predicate map") + entries = [(n, p) for n, p in name_or_map.items() if callable(p)] + for n, _ in entries: + self._assert_known(n) + return entries + + def define_situations(self, situations: Mapping[str, SituationConfig]) -> ToolSet: + """Register named declarative situations. + + Each situation overlays the base partition -- ids it does not mention + keep whatever the base set declares. Replaces any previously defined + situations (last-call-wins at the registry level). + """ + next_situations: Dict[str, _SituationRuntime] = {} + for name, config in situations.items(): + enabled = list(config.get("enabled") or []) + disabled = list(config.get("disabled") or []) + conditional_entries = [ + (tid, rule) for tid, rule in (config.get("conditional") or {}).items() if rule is not None + ] + + seen: set[str] = set() + + def record(tool_id: str) -> None: + self._assert_known(tool_id) + if tool_id in seen: + raise ValueError( + f'Situation "{name}" lists tool "{tool_id}" more than once (across enabled/disabled/conditional)' + ) + seen.add(tool_id) + + for tid in enabled: + record(tid) + for tid in disabled: + record(tid) + for tid, _ in conditional_entries: + record(tid) + + conditional: List[Tuple[str, ConditionalMode, ActivationPredicate]] = [] + for tid, rule in conditional_entries: + mode, pred = _normalize_conditional_rule(name, tid, rule) + conditional.append((tid, mode, pred)) + + next_situations[name] = _SituationRuntime( + enabled=tuple(enabled), disabled=tuple(disabled), conditional=tuple(conditional) + ) + + if self._mutable: + self._situations.clear() + self._situations.update(next_situations) + return self + return ToolSet(self._index, dict(self._activation), next_situations, False) + + def resolve(self, input: Optional[ActivationInput] = None) -> ResolvedToolSnapshot: + """Resolve against the base partition (no situation overlay).""" + return self._resolve_with_activation(self._activation, input) + + def infer_tools(self, input: Optional[ActivationInput] = None) -> Dict[Any, Any]: + """Back-compat alias for :meth:`resolve`. Prefer ``resolve`` for new code. + + Returns a ``dict`` with ``tools``, ``active_tools``, ``enabled``, + ``disabled``, ``status_by_tool`` and the :data:`TOOL_SET_SNAPSHOT` + marker. Only ``tools`` and ``active_tools`` are meant to reach + ``call_model``; the marker lets :func:`strip_tool_set_snapshot_metadata` + drop the rest when the whole dict is spread into a request. + """ + snapshot = self.resolve(input) + return { + "tools": list(snapshot.tools), + "active_tools": list(snapshot.active_tools), + "enabled": snapshot.enabled, + "disabled": snapshot.disabled, + "status_by_tool": snapshot.status_by_tool, + TOOL_SET_SNAPSHOT: True, + } + + def resolve_situation(self, name: str, input: Optional[ActivationInput] = None) -> ResolvedToolSnapshot: + """Resolve a previously defined named situation.""" + situation = self._situations.get(name) + if situation is None: + raise ValueError(f'Unknown situation: "{name}"') + + activation = dict(self._activation) + for tid in situation.enabled: + activation[tid] = _ActivationEntry(kind="static", active=True, source="situation") + for tid in situation.disabled: + activation[tid] = _ActivationEntry(kind="static", active=False, source="situation") + for tid, mode, pred in situation.conditional: + activation[tid] = _ActivationEntry(kind=mode, predicate=pred, source="situation") + return self._resolve_with_activation(activation, input) + + def _resolve_with_activation( + self, activation: Mapping[str, _ActivationEntry], input: Optional[ActivationInput] + ) -> ResolvedToolSnapshot: + resolved_input: ActivationInput = input if input is not None else {} + tools: List[Any] = [] + active_tools: List[str] = [] + enabled: List[str] = [] + disabled: List[str] = [] + status_by_tool: Dict[str, ToolStatusEntry] = {} + + for tid in self._index.ordered_ids: + tool = self._index.tool_by_id.get(tid) + if tool is None: + continue + entry = activation.get(tid) + active = self._evaluate(entry, resolved_input) + status_by_tool[tid] = self._to_status_entry(active, entry) + if active: + tools.append(tool) + enabled.append(tid) + if not is_server_tool(tool): + active_tools.append(tid) + else: + disabled.append(tid) + + return ResolvedToolSnapshot( + tools=tools, + active_tools=active_tools, + call_model={"tools": tools, "active_tools": active_tools}, + enabled=enabled, + disabled=disabled, + status_by_tool=status_by_tool, + ) + + @staticmethod + def _evaluate(entry: Optional[_ActivationEntry], input: ActivationInput) -> bool: + if entry is None: + return True + if entry.kind == "static": + return entry.active + assert entry.predicate is not None + if entry.kind == "activateWhen": + return entry.predicate(input) is True + return entry.predicate(input) is not True + + @staticmethod + def _to_status_entry(active: bool, entry: Optional[_ActivationEntry]) -> ToolStatusEntry: + if entry is None: + return {"enabled": active, "reason": "default"} + if entry.kind == "static": + directive: Directive = "activate" if entry.active else "deactivate" + reason: StatusReason + if entry.source == "situation": + reason = "situation" + elif entry.source == "default": + reason = "default" + else: + reason = directive + return {"enabled": active, "reason": reason, "directive": directive} + cond_directive: Directive = entry.kind + cond_reason: StatusReason = "situation" if entry.source == "situation" else cond_directive + return {"enabled": active, "reason": cond_reason, "directive": cond_directive, "predicate": True} + + def clone(self, *, mutable: Optional[bool] = None) -> ToolSet: + """Copy state into a fresh, independent instance. + + ``mutable=None`` inherits the source's mode. + """ + next_mutable = self._mutable if mutable is None else bool(mutable) + return ToolSet(self._index, dict(self._activation), dict(self._situations), next_mutable) + + +def create_tool_set(tools: Sequence[Any], *, mutable: bool = False) -> ToolSet: + """Construct a :class:`ToolSet` (upstream ``createToolSet``). + + Every tool starts enabled. Raises ``ValueError`` on duplicate tool ids. + """ + return ToolSet.create(tools=tools, mutable=mutable) + + +__all__ = [ + "TOOL_SET_SNAPSHOT", + "ActivationInput", + "ActivationPredicate", + "ConditionalMode", + "Directive", + "ResolvedToolSnapshot", + "SituationConditionalRule", + "SituationConditionalRuleObject", + "SituationConfig", + "StatusReason", + "ToolSet", + "ToolStatusEntry", + "create_tool_set", + "get_server_tool_id", + "get_tool_id", + "strip_tool_set_snapshot_metadata", +] diff --git a/src/openrouter_agent/tool_types.py b/src/openrouter_agent/tool_types.py index f6db4b9..c47cd42 100644 --- a/src/openrouter_agent/tool_types.py +++ b/src/openrouter_agent/tool_types.py @@ -24,6 +24,12 @@ class ParsedToolCall: id: str name: str arguments: Any + #: Set on persisted `pending_tool_calls` (upstream `PendingToolCall. + #: preToolUseApplied`): PreToolUse already ran during approval + #: preparation and `arguments` holds its effective (possibly mutated) + #: input, so the approval resume must not run the hook again. Defaults to + #: False so legacy state blobs without the field still load. + pre_tool_use_applied: bool = False @dataclass(frozen=True) @@ -57,6 +63,12 @@ class ConversationState: #: Optional so legacy (pre-version-field) states remain constructible; #: absence is treated as version 1 by `deserialize_conversation_state`. version: Optional[int] = None + #: Semantic identity of a forced `tool_choice` that already produced a + #: tool call in this logical run, so a resume keeps it relaxed to `auto`. + consumed_forced_tool_choice_key: Optional[str] = None + #: Doom-loop detector snapshot (see `openrouter_agent.doom_loop`), present + #: only when detection is enabled. + doom_loop: Optional[Dict[str, Any]] = None @dataclass(frozen=True) @@ -131,7 +143,6 @@ class ToolExecutionResult(TypedDict, total=False): #: locally-defined tools. See `is_mcp_tool` / `mark_mcp`. source: Literal["client", "mcp"] result: object - preliminary_results: List[object] error: BaseException @@ -149,6 +160,8 @@ class ToolPreliminaryResultEvent(TypedDict, total=False): type: Literal["tool.preliminary_result", "preliminary_result"] toolCallId: str tool_call_id: str + toolName: str + tool_name: str result: object timestamp: int @@ -157,12 +170,12 @@ class ToolResultEvent(TypedDict, total=False): type: Literal["tool.result", "tool_result"] toolCallId: str tool_call_id: str + toolName: str + tool_name: str #: Origin of the tool result: "mcp" for MCP-branded tools, "client" #: otherwise. See `is_mcp_tool` / `mark_mcp`. source: Literal["client", "mcp"] result: object - preliminaryResults: List[object] - preliminary_results: List[object] timestamp: int diff --git a/src/openrouter_agent/turn_context.py b/src/openrouter_agent/turn_context.py index 6d991a3..840fe12 100644 --- a/src/openrouter_agent/turn_context.py +++ b/src/openrouter_agent/turn_context.py @@ -2,6 +2,8 @@ from typing import Any, Dict, List +from ._utils import get_field + def normalize_input_to_array(value: Any) -> List[Any]: if value is None: @@ -20,3 +22,22 @@ def build_turn_context(number_of_turns: int = 0, tool_call: Any = None, turn_req if turn_request is not None: ctx["turn_request"] = turn_request return ctx + + +def is_input_item(value: Any) -> bool: + """Whether ``value`` is shaped like a single item of a Responses ``input`` + array: an object carrying a string item ``type`` or message ``role``. + + Response output items are fed back as input on the next turn; this guard is + how an output item crosses over. Accepts mappings and SDK model objects. + """ + if value is None or isinstance(value, (str, bytes, int, float, bool, list, tuple)): + return False + return isinstance(get_field(value, "type"), str) or isinstance(get_field(value, "role"), str) + + +def output_to_input_items(output: Any) -> List[Any]: + """Convert a response's output into input items for the next request, + dropping anything that is not item-shaped.""" + items = output if isinstance(output, (list, tuple)) else [output] + return [item for item in items if is_input_item(item)] diff --git a/src/openrouter_agent/user_agent.py b/src/openrouter_agent/user_agent.py new file mode 100644 index 0000000..e5cab33 --- /dev/null +++ b/src/openrouter_agent/user_agent.py @@ -0,0 +1,68 @@ +"""Agent SDK user-agent token. + +Ports `packages/agent/src/lib/user-agent.ts`. Keeps the generated SDK's base +user-agent string byte-identical and appends one token identifying this +package for OpenRouter analytics. + +Deliberate divergence (contract, Idiomatic Divergences #7): the token is the +PyPI distribution name ``openrouter-agent-sdk``, not upstream's npm name +``@openrouter/agent``. Format matches upstream: `` /``. +""" + +from __future__ import annotations + +from importlib import metadata +from typing import Callable, Optional + +from openrouter._version import __user_agent__ as SDK_USER_AGENT + +AGENT_PACKAGE_NAME = "openrouter-agent-sdk" + + +def get_package_version(version_getter: Optional[Callable[[str], str]] = None) -> Optional[str]: + """This distribution's installed version, or ``None`` when no distribution + metadata is available (e.g. running from a bare source checkout).""" + try: + return (version_getter or metadata.version)(AGENT_PACKAGE_NAME) + except metadata.PackageNotFoundError: + return None + + +def agent_user_agent_token(version_getter: Optional[Callable[[str], str]] = None) -> str: + """``openrouter-agent-sdk/``; the bare name if the version is unknown.""" + version = get_package_version(version_getter) + if version is None: + return AGENT_PACKAGE_NAME + return f"{AGENT_PACKAGE_NAME}/{version}" + + +def append_agent_user_agent(user_agent: str, agent_token: str) -> str: + """Append ``agent_token`` to ``user_agent`` once (idempotent).""" + if agent_token in user_agent.split(): + return user_agent + if not user_agent: + return agent_token + return f"{user_agent} {agent_token}" + + +def build_agent_user_agent(version_getter: Optional[Callable[[str], str]] = None) -> str: + return append_agent_user_agent(SDK_USER_AGENT, agent_user_agent_token(version_getter)) + + +PACKAGE_VERSION: Optional[str] = get_package_version() +"""This package's version as recorded in installed distribution metadata.""" + +AGENT_USER_AGENT: str = build_agent_user_agent() +"""The default user agent the `OpenRouter` client sends, e.g. +``speakeasy-sdk/python 1.x ... openrouter-agent-sdk/0.8.0``.""" + +__all__ = [ + "AGENT_PACKAGE_NAME", + "AGENT_USER_AGENT", + "PACKAGE_VERSION", + "SDK_USER_AGENT", + "agent_user_agent_token", + "append_agent_user_agent", + "build_agent_user_agent", + "get_package_version", +] diff --git a/tests/unit/test_approval_gate_regressions.py b/tests/unit/test_approval_gate_regressions.py new file mode 100644 index 0000000..29644aa --- /dev/null +++ b/tests/unit/test_approval_gate_regressions.py @@ -0,0 +1,1630 @@ +"""Port of upstream `tests/unit/approval-gate-regressions.test.ts`. + +Pins the approval-gate regression invariants: + +- approval predicates see the same schema-parsed arguments `execute` gets, + and schema-invalid arguments fail closed; +- every engine execute path validates before running the tool body; +- the `allow_final_response` path (stop_when halting mid-tool-call) runs the + approval gate, and each call occurrence is gated at most once per run; +- approve/reject decisions are de-duplicated and validated before anything + runs, and a paused conversation resumed without decisions stays paused. + +Upstream drives `ModelResult` with a mocked `betaResponsesSend`; this port +drives `call_model` with the shared `QueuedClient`. Upstream cases that depend +on engine features this port does not implement (unified `run` tools) are kept +as named `skip`s so the gap stays visible. +""" + +from __future__ import annotations + +import copy +import dataclasses +import json +import re +from typing import Any, AsyncIterator, Awaitable, Callable, Dict, List, Optional, Sequence, Tuple + +import pytest +from pydantic import BaseModel, field_validator + +from openrouter_agent import HookEntry, HookName, HooksManager, call_model, step_count_is, tool +from openrouter_agent.conversation_state import tool_requires_approval +from openrouter_agent.tool_executor import execute_generator_tool, execute_hitl_tool, execute_regular_tool +from openrouter_agent.tool_types import ParsedToolCall, Tool +from tests._fixtures import MemoryStateAccessor, QueuedClient, function_call_item, make_response, text_response + +CONTEXT: Dict[str, Any] = {"number_of_turns": 1} + + +class Recorder: + """Counts invocations and records arguments; returns a fixed value. + + A class (not `lambda: calls.append(...) or value`) so the return value is + explicit and mypy-clean -- see tests/unit/test_tool_execution_once.py. + """ + + def __init__(self, value: Any = None) -> None: + self.value = value + self.calls: List[Tuple[Any, ...]] = [] + + def __call__(self, *args: Any) -> Any: + self.calls.append(args) + return self.value + + @property + def count(self) -> int: + return len(self.calls) + + +def _dumped_first_args(recorder: Recorder) -> List[Any]: + return [call[0].model_dump() if isinstance(call[0], BaseModel) else call[0] for call in recorder.calls] + + +def _function_call_outputs(client: QueuedClient, index: int) -> List[Dict[str, Any]]: + sent = client.requests[index].get("input") or [] + return [item for item in sent if isinstance(item, dict) and item.get("type") == "function_call_output"] + + +def _deep_nested(levels: int = 200) -> Dict[str, Any]: + deep: Dict[str, Any] = {} + for _ in range(levels): + deep = {"child": deep} + return deep + + +def _done(response_id: str, text: str = "Done.") -> Dict[str, Any]: + return text_response(response_id, text) + + +# --------------------------------------------------------------------------- +# Bug 1: the approval predicate must see the SAME arguments `execute` gets. +# --------------------------------------------------------------------------- + + +class DangerousDefaultArgs(BaseModel): + dangerous: bool = True + + +class AmountArgs(BaseModel): + amount: int + + +class DestructiveDefaultArgs(BaseModel): + destructive: bool = True + + +class TargetArgs(BaseModel): + target: str + + +class OkOutput(BaseModel): + ok: bool + + +async def test_applies_input_schema_defaults_before_function_require_approval() -> None: + predicate = Recorder() + + def require(params: Any, ctx: Any) -> bool: + predicate(params, ctx) + return params["dangerous"] is True + + conditional = tool( + name="conditional_action", + input_schema=DangerousDefaultArgs, + require_approval=require, + execute=lambda params, ctx: {}, + ) + + # The model emitted `{}`; the schema default makes `dangerous` True by the + # time execute runs, so the predicate must see it True too. + requires = await tool_requires_approval( + ParsedToolCall(id="1", name="conditional_action", arguments={}), [conditional], CONTEXT + ) + + # Pydantic models are model_dump()'d for the predicate (conversation_state.py + # _parse_tool_arguments): the predicate contract is a mapping, as upstream. + assert [call[0] for call in predicate.calls] == [{"dangerous": True}] + assert requires is True + + +async def test_applies_input_schema_coercions_before_function_require_approval() -> None: + predicate = Recorder() + + def require(params: Any, ctx: Any) -> bool: + predicate(params, ctx) + return bool(params["amount"] > 100) + + transfer = tool( + name="transfer", + input_schema=AmountArgs, + require_approval=require, + execute=lambda params, ctx: {}, + ) + + # String amount is coerced (Pydantic lax mode stands in for z.coerce), so + # the comparison is numeric, not lexicographic. + requires = await tool_requires_approval( + ParsedToolCall(id="2", name="transfer", arguments={"amount": "500"}), [transfer], CONTEXT + ) + + assert [call[0] for call in predicate.calls] == [{"amount": 500}] + assert requires is True + + +async def test_applies_schema_defaults_for_call_level_checks_without_mutating_call() -> None: + defaulted = tool( + name="call_level_defaulted_action", + input_schema=DestructiveDefaultArgs, + execute=lambda params, ctx: {}, + ) + tool_call = ParsedToolCall(id="call-level-default", name="call_level_defaulted_action", arguments={}) + call_level_check = Recorder(True) + + requires = await tool_requires_approval(tool_call, [defaulted], CONTEXT, call_level_check) + + assert call_level_check.calls == [(dataclasses.replace(tool_call, arguments={"destructive": True}), CONTEXT)] + assert tool_call.arguments == {} + assert requires is True + + +@pytest.mark.parametrize("decision", [False, True]) +async def test_honors_call_level_check_when_arguments_fail_schema_validation(decision: bool) -> None: + strict = tool(name="call_level_invalid_action", input_schema=TargetArgs, execute=lambda params, ctx: {}) + invalid_call = ParsedToolCall(id=f"call-level-invalid-{decision}", name="call_level_invalid_action", arguments={}) + call_level_check = Recorder(decision) + + assert await tool_requires_approval(invalid_call, [strict], CONTEXT, call_level_check) is decision + # Parsing failed, so the call-level check receives the raw call. + assert call_level_check.calls == [(invalid_call, CONTEXT)] + + +async def test_parses_original_wire_arguments_once_for_predicate_and_once_for_execution() -> None: + transform_calls = {"n": 0} + + class TransformedArgs(BaseModel): + value: str + + @field_validator("value") + @classmethod + def _suffix(cls, value: str) -> str: + transform_calls["n"] += 1 + return f"{value}:{transform_calls['n']}" + + predicate = Recorder(False) + execute = Recorder({}) + transformed = tool( + name="transformed_action", + input_schema=TransformedArgs, + require_approval=predicate, + execute=execute, + ) + tool_call = ParsedToolCall(id="non-idempotent-transform", name="transformed_action", arguments={"value": "wire"}) + + assert await tool_requires_approval(tool_call, [transformed], CONTEXT) is False + await execute_regular_tool(transformed, tool_call, CONTEXT) + + assert predicate.calls == [({"value": "wire:1"}, CONTEXT)] + assert _dumped_first_args(execute) == [{"value": "wire:2"}] + assert transform_calls["n"] == 2 + assert tool_call.arguments == {"value": "wire"} + + +async def test_fails_closed_when_arguments_fail_schema_validation() -> None: + predicate = Recorder(False) + strict = tool(name="strict_action", input_schema=TargetArgs, require_approval=predicate, execute=lambda p, c: {}) + + requires = await tool_requires_approval( + ParsedToolCall(id="3", name="strict_action", arguments={}), [strict], CONTEXT + ) + + assert requires is True + assert predicate.count == 0 + + +async def test_still_fails_closed_when_schema_parses_to_non_object_value() -> None: + predicate = Recorder(False) + weird = tool(name="weird_action", input_schema=TargetArgs, require_approval=predicate, execute=lambda p, c: {}) + # Mirrors upstream swapping in a schema whose parse succeeds with a + # non-record payload: a callable schema is honored by validate_schema. + weird["function"]["input_schema"] = lambda value: ["not", "a", "record"] + + requires = await tool_requires_approval( + ParsedToolCall(id="4", name="weird_action", arguments={"target": "x"}), [weird], CONTEXT + ) + + assert requires is True + assert predicate.count == 0 + + +async def test_still_fails_closed_on_schema_invalid_arguments_for_manual_tools() -> None: + predicate = Recorder(False) + # A manual tool: no execute / on_tool_called. The host runs it as-is, so a + # malformed call must not be waved past the gate. + manual_tool: Tool = { + "type": "function", + "function": {"name": "manual_action", "input_schema": TargetArgs, "require_approval": predicate}, + } + + requires = await tool_requires_approval( + ParsedToolCall(id="5", name="manual_action", arguments={}), [manual_tool], CONTEXT + ) + + assert requires is True + assert predicate.count == 0 + + +# --------------------------------------------------------------------------- +# Approval vs. PreToolUse-mutated arguments. +# --------------------------------------------------------------------------- + + +class GuardedArgs(BaseModel): + dangerous: bool + + +async def test_does_not_let_a_hook_rewrite_safe_arguments_into_unapproved_dangerous_call() -> None: + predicate = Recorder() + + def require(params: Any, ctx: Any) -> bool: + predicate(params, ctx) + return bool(params["dangerous"]) + + execute = Recorder({"ok": True}) + guarded = tool( + name="guarded_action", + input_schema=GuardedArgs, + output_schema=OkOutput, + require_approval=require, + execute=execute, + ) + pre_tool_use = Recorder({"mutated_input": {"dangerous": True}}) + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=pre_tool_use)) + accessor = MemoryStateAccessor() + client = QueuedClient( + [make_response("resp_mutation", [function_call_item("call_guarded", "guarded_action", '{"dangerous":false}')])] + ) + + await call_model( + client, + { + "model": "test-model", + "input": "run the guarded action", + "tools": [guarded], + "hooks": hooks, + "state": accessor, + }, + ).get_response() + + # Initial gate on the wire arguments, then the mutated-input re-gate. + # Upstream asserts `toHaveBeenCalledWith(mutatedInput, ...)`; the full + # ordered call list pins that the re-gate is the second (and last) check. + assert predicate.calls == [ + ({"dangerous": False}, {"number_of_turns": 0}), + ({"dangerous": True}, {"number_of_turns": 0}), + ] + assert pre_tool_use.count == 1 + assert execute.count == 0 + assert len(client.requests) == 1 + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + assert accessor.stored.pending_tool_calls == [ + ParsedToolCall( + id="call_guarded", name="guarded_action", arguments={"dangerous": True}, pre_tool_use_applied=True + ) + ] + + +async def test_does_not_abort_approval_bookkeeping_for_circular_hook_mutations() -> None: + circular: Dict[str, Any] = {} + circular["self"] = circular + + class SelfArgs(BaseModel): + # Pydantic v2's __init__ takes `self` positional-only, so a field + # named `self` mirrors upstream's `z.object({ self: z.unknown() })`. + self: Any = None + + predicate = Recorder(False) + execute = Recorder({"ok": True}) + guarded = tool( + name="circular_mutation", + input_schema=SelfArgs, + output_schema=OkOutput, + require_approval=predicate, + execute=execute, + ) + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=lambda p, c: {"mutated_input": circular})) + client = QueuedClient( + [ + make_response( + "resp_circular_mutation", [function_call_item("call_circular_mutation", "circular_mutation", "{}")] + ), + # Upstream's `mockResolvedValue` repeats the empty response; the + # port's empty-final tolerance retries once, so queue two. + make_response("resp_circular_done", []), + make_response("resp_circular_done_retry", []), + ] + ) + + response = await call_model( + client, {"model": "test-model", "input": "run", "tools": [guarded], "hooks": hooks} + ).get_response() + + assert response["id"] == "resp_circular_done_retry" + # The mutated-input re-gate received the circular input (dict equality + # short-circuits on the self-reference identity). + assert predicate.calls[-1] == (circular, {"number_of_turns": 0}) + assert predicate.count == 2 + assert execute.count == 0 + + +async def test_lets_normal_validation_handle_deeply_nested_model_input() -> None: + execute = Recorder({"ok": True}) + strict = tool( + name="deep_invalid", + input_schema=TargetArgs, + output_schema=OkOutput, + require_approval=lambda params, ctx: False, + execute=execute, + ) + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + make_response( + "resp_deep_invalid", + [function_call_item("call_deep_invalid", "deep_invalid", json.dumps(_deep_nested()))], + ) + ] + ) + + response = await call_model( + client, {"model": "test-model", "input": "run", "tools": [strict], "state": accessor} + ).get_response() + + assert response["id"] == "resp_deep_invalid" + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + assert execute.count == 0 + assert len(client.requests) == 1 + + +async def test_gates_identical_post_hook_calls_independently_across_responses() -> None: + predicate = Recorder() + + def require(params: Any, ctx: Any) -> bool: + predicate(params, ctx) + return bool(params["dangerous"]) + + execute = Recorder({"ok": True}) + guarded = tool( + name="guarded_action", + input_schema=GuardedArgs, + output_schema=OkOutput, + require_approval=require, + execute=execute, + ) + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=lambda p, c: {"mutated_input": {"dangerous": True}})) + permission_request = Recorder({"decision": "allow"}) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_request)) + # Two distinct response objects with identical ids and calls. + client = QueuedClient( + [ + make_response( + "resp_mutation_reused", [function_call_item("call_reused", "guarded_action", '{"dangerous":false}')] + ), + make_response( + "resp_mutation_reused", [function_call_item("call_reused", "guarded_action", '{"dangerous":false}')] + ), + _done("resp_done"), + ] + ) + + await call_model( + client, {"model": "test-model", "input": "run twice", "tools": [guarded], "hooks": hooks} + ).get_response() + + assert [call[0] for call in predicate.calls if call[0]["dangerous"] is True] == [ + {"dangerous": True}, + {"dangerous": True}, + ] + assert permission_request.count == 2 + assert [call[0]["tool_input"] for call in permission_request.calls] == [{"dangerous": True}, {"dangerous": True}] + assert execute.count == 2 + assert len(client.requests) == 3 + + +async def test_keeps_hook_mutated_arguments_raw_across_approval_and_execution_transforms() -> None: + class NormalizedArgs(BaseModel): + value: str + + @field_validator("value") + @classmethod + def _normalize(cls, value: str) -> str: + return f"{value}:normalized" + + class ValueOutput(BaseModel): + value: str + + predicate = Recorder() + + def require(params: Any, ctx: Any) -> bool: + predicate(params, ctx) + return bool(params["value"] == "hook:normalized") + + execute = Recorder() + + def run(params: NormalizedArgs, ctx: Any) -> Dict[str, Any]: + execute(params, ctx) + return {"value": params.value} + + guarded = tool( + name="transformed_mutation", + input_schema=NormalizedArgs, + output_schema=ValueOutput, + require_approval=require, + execute=run, + ) + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=lambda p, c: {"mutated_input": {"value": "hook"}})) + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + make_response( + "resp_transformed_mutation", + [function_call_item("call_transformed_mutation", "transformed_mutation", '{"value":"wire"}')], + ) + ] + ) + + await call_model( + client, {"model": "test-model", "input": "run", "tools": [guarded], "hooks": hooks, "state": accessor} + ).get_response() + + # The predicate judges the schema-transformed post-hook input... + assert predicate.calls[-1] == ({"value": "hook:normalized"}, {"number_of_turns": 0}) + assert execute.count == 0 + # ...but the pending call keeps the RAW post-hook input, so execution + # transforms it exactly once. + assert accessor.stored is not None + assert accessor.stored.pending_tool_calls == [ + ParsedToolCall( + id="call_transformed_mutation", + name="transformed_mutation", + arguments={"value": "hook"}, + pre_tool_use_applied=True, + ) + ] + + +async def test_executes_a_type_changing_hook_mutation_from_raw_input_exactly_once() -> None: + class LengthArgs(BaseModel): + value: int + + @field_validator("value", mode="before") + @classmethod + def _length(cls, value: Any) -> int: + if not isinstance(value, str): + raise ValueError("value must be a string") + return len(value) + + class LengthOutput(BaseModel): + value: int + + execute = Recorder() + + def run(params: LengthArgs, ctx: Any) -> Dict[str, Any]: + execute(params, ctx) + return {"value": params.value} + + transformed = tool( + name="type_changing_mutation", + input_schema=LengthArgs, + output_schema=LengthOutput, + require_approval=lambda params, ctx: False, + execute=run, + ) + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=lambda p, c: {"mutated_input": {"value": "hook"}})) + client = QueuedClient( + [ + make_response( + "resp_type_changing_mutation", + [function_call_item("call_type_changing_mutation", "type_changing_mutation", '{"value":"wire"}')], + ), + # Upstream's `mockResolvedValue` repeats the empty response; the + # port's empty-final tolerance retries once, so queue two. + make_response("resp_type_changing_done", []), + make_response("resp_type_changing_done_retry", []), + ] + ) + + await call_model( + client, {"model": "test-model", "input": "run", "tools": [transformed], "hooks": hooks} + ).get_response() + + assert execute.count == 1 + # Executed from the hook's raw `{"value": "hook"}` -> len("hook") == 4. + assert _dumped_first_args(execute) == [{"value": 4}] + + +async def test_gates_schema_invalid_model_arguments_before_running_pre_tool_use() -> None: + class DangerousArgs(BaseModel): + dangerous: bool + + pre_tool_use = Recorder({"mutated_input": {"dangerous": True}}) + execute = Recorder({"ok": True}) + guarded = tool( + name="guarded_action", + input_schema=DangerousArgs, + require_approval=lambda params, ctx: params["dangerous"], + execute=execute, + ) + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=pre_tool_use)) + accessor = MemoryStateAccessor() + client = QueuedClient( + [make_response("resp_invalid_original", [function_call_item("call_invalid_original", "guarded_action", "{}")])] + ) + + await call_model( + client, {"model": "test-model", "input": "run", "tools": [guarded], "hooks": hooks, "state": accessor} + ).get_response() + + assert pre_tool_use.count == 0 + assert execute.count == 0 + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + assert len(client.requests) == 1 + + +async def test_persists_unconditional_approvals_and_does_not_reapply_hook_on_resume() -> None: + class ValueArgs(BaseModel): + value: str + + class ValueOutput(BaseModel): + value: str + + pre_tool_use = Recorder() + + def prepare(payload: Dict[str, Any], ctx: Any) -> Dict[str, Any]: + pre_tool_use(payload, ctx) + return {"mutated_input": {"value": f"{payload['tool_input']['value']}-prepared"}} + + permission_request = Recorder({"decision": "ask_user"}) + execute = Recorder() + + def run(params: ValueArgs, ctx: Any) -> Dict[str, Any]: + execute(params, ctx) + return {"value": params.value} + + guarded = tool( + name="always_guarded", + input_schema=ValueArgs, + output_schema=ValueOutput, + require_approval=True, + execute=run, + ) + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=prepare)) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_request)) + accessor = MemoryStateAccessor() + + def request(state: MemoryStateAccessor, **extra: Any) -> Dict[str, Any]: + return { + "model": "test-model", + "input": "run the guarded action", + "tools": [guarded], + "hooks": hooks, + "state": state, + **extra, + } + + first = QueuedClient( + [ + make_response( + "resp_unconditional", + [function_call_item("call_unconditional", "always_guarded", '{"value":"original"}')], + ) + ] + ) + await call_model(first, request(accessor)).get_response() + + assert permission_request.count == 1 + assert permission_request.calls[0][0]["tool_input"] == {"value": "original"} + assert accessor.stored is not None + assert accessor.stored.pending_tool_calls == [ + ParsedToolCall(id="call_unconditional", name="always_guarded", arguments={"value": "original"}) + ] + # Upstream deletes `preToolUseApplied` to build a "legacy" state; the + # port's equivalent is the default (False) marker. + legacy_state = copy.deepcopy(accessor.stored) + legacy_state = dataclasses.replace( + legacy_state, + pending_tool_calls=[ + dataclasses.replace(call, pre_tool_use_applied=False) for call in legacy_state.pending_tool_calls or [] + ], + ) + + resume = QueuedClient([_done("resp_after_approval")]) + await call_model(resume, request(accessor, approve_tool_calls=["call_unconditional"])).get_response() + + assert pre_tool_use.count == 1 + assert permission_request.count == 1 + assert execute.count == 1 + assert _dumped_first_args(execute) == [{"value": "original-prepared"}] + assert len(resume.requests) == 1 + + legacy_accessor = MemoryStateAccessor() + legacy_accessor.stored = legacy_state + legacy_resume = QueuedClient([_done("resp_after_legacy_approval")]) + await call_model(legacy_resume, request(legacy_accessor, approve_tool_calls=["call_unconditional"])).get_response() + + assert pre_tool_use.count == 2 + assert execute.count == 2 + assert _dumped_first_args(execute)[-1] == {"value": "original-prepared"} + + +# --------------------------------------------------------------------------- +# Every engine execute path validates before running the tool body. +# --------------------------------------------------------------------------- + + +async def test_regular_execute_tools_validate_before_running() -> None: + execute = Recorder({}) + regular = tool(name="regular", input_schema=TargetArgs, execute=execute) + + result = await execute_regular_tool(regular, ParsedToolCall(id="c1", name="regular", arguments={}), CONTEXT) + + assert execute.count == 0 + assert result.get("error") is not None + + +async def test_generator_tools_validate_before_running() -> None: + started = Recorder() + + class Progress(BaseModel): + progress: int + + class Done(BaseModel): + done: bool + + async def run(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + started(params, ctx) + yield {"progress": 1} + yield {"done": True} + + generator = tool(name="generator", input_schema=TargetArgs, event_schema=Progress, output_schema=Done, execute=run) + + result = await execute_generator_tool(generator, ParsedToolCall(id="c2", name="generator", arguments={}), CONTEXT) + + assert started.count == 0 + assert result.get("error") is not None + + +async def test_hitl_on_tool_called_tools_validate_before_running() -> None: + on_tool_called = Recorder({}) + hitl = tool(name="hitl", input_schema=TargetArgs, output_schema=OkOutput, on_tool_called=on_tool_called) + + result = await execute_hitl_tool(hitl, ParsedToolCall(id="c3", name="hitl", arguments={}), CONTEXT) + + assert on_tool_called.count == 0 + assert result is not None + assert result.get("error") is not None + + +@pytest.mark.skip( + reason="Not implemented in the Python port: unified `run` tools / prepareUnifiedInvocation " + "(tool() has no `run` parameter; tool.py)." +) +async def test_unified_run_tools_validate_before_running() -> None: + """Upstream: prepareUnifiedInvocation returns an error without calling `run`.""" + + +# --------------------------------------------------------------------------- +# Bug 2: the allow_final_response path enforces the approval gate (#54). +# --------------------------------------------------------------------------- + + +def _safe_and_danger_tools() -> Tuple[Recorder, Recorder, List[Tool]]: + safe_execute = Recorder({"ok": True}) + danger_execute = Recorder({"ok": True}) + safe = tool(name="safe", input_schema=TargetArgs, output_schema=OkOutput, execute=safe_execute) + danger = tool( + name="danger", input_schema=TargetArgs, output_schema=OkOutput, require_approval=True, execute=danger_execute + ) + return safe_execute, danger_execute, [safe, danger] + + +def _safe_then_danger_client(final_text: str = "Done.") -> QueuedClient: + return QueuedClient( + [ + make_response("resp_turn_0", [function_call_item("call_safe", "safe", '{"target":"staging"}')]), + make_response("resp_turn_1", [function_call_item("call_danger", "danger", '{"target":"prod"}')]), + _done("resp_final", final_text), + ] + ) + + +async def _assert_gate_holds_on_allow_final_response_path(stop_when: Any) -> None: + safe_execute, danger_execute, tools = _safe_and_danger_tools() + client = _safe_then_danger_client() + accessor = MemoryStateAccessor() + + result = call_model( + client, + { + "model": "test-model", + "input": "do the thing", + "tools": tools, + "state": accessor, + "stop_when": stop_when, + "allow_final_response": True, + }, + ) + await result.get_response() + + # The ungated tool ran in the normal loop round, exactly once. + assert safe_execute.count == 1 + # The gate holds on the allow_final_response path: the gated tool never + # runs without approval, and the run pauses like the in-loop gate. + assert danger_execute.count == 0 + saved = accessor.stored + assert saved is not None + assert saved.status == "awaiting_approval" + assert [call.id for call in saved.pending_tool_calls or []] == ["call_danger"] + assert await result.requires_approval() is True + # No final text-coercion request while paused: initial + one follow-up. + assert len(client.requests) == 2 + + +async def test_does_not_execute_require_approval_tool_when_stop_when_halts_the_loop() -> None: + # Stop conditions see executed tool rounds (upstream shouldStopExecution): + # step_count_is(1) fires on turn 1's response, the one carrying the gated + # call, which reaches the post-loop allow_final_response path. + await _assert_gate_holds_on_allow_final_response_path(step_count_is(1)) + + +async def test_still_runs_non_gated_tools_on_the_allow_final_response_path() -> None: + execute = Recorder({"ok": True}) + safe = tool(name="safe", input_schema=TargetArgs, output_schema=OkOutput, execute=execute) + client = QueuedClient( + [ + make_response("resp_turn_0", [function_call_item("call_safe", "safe", '{"target":"staging"}')]), + _done("resp_final"), + ] + ) + + text = await call_model( + client, + { + "model": "test-model", + "input": "do the safe thing", + "tools": [safe], + "state": MemoryStateAccessor(), + "stop_when": step_count_is(0), + "allow_final_response": True, + }, + ).get_text() + + assert execute.count == 1 + assert text == "Done." + assert len(client.requests) == 2 + + +async def _assert_permission_deny_on_allow_final_response_path(stop_when: Any) -> None: + safe_execute, danger_execute, tools = _safe_and_danger_tools() + hooks = HooksManager() + permission_handler = Recorder({"decision": "deny", "reason": "blocked by policy"}) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_handler)) + client = _safe_then_danger_client("Denied.") + accessor = MemoryStateAccessor() + + await call_model( + client, + { + "model": "test-model", + "input": "do the thing", + "tools": tools, + "hooks": hooks, + "state": accessor, + "stop_when": stop_when, + "allow_final_response": True, + }, + ).get_response() + + # The hook was consulted for the gated call exactly once. + assert permission_handler.count == 1 + assert danger_execute.count == 0 + assert safe_execute.count == 1 + # A hook deny resolves the gate without a human: no approval pause. + saved = accessor.stored + assert saved is not None + assert saved.status != "awaiting_approval" + # The rejection is recorded in state as a synthesized output carrying the + # hook's reason. State items keep upstream's camelCase `callId` + # (model_result.py _rejected_output); only _send converts to `call_id`. + outputs = [ + message + for message in saved.messages + if isinstance(message, dict) + and message.get("type") == "function_call_output" + and message.get("callId") == "call_danger" + ] + assert len(outputs) == 1 + assert "blocked by policy" in json.dumps(outputs[0]["output"]) + + +async def test_honors_permission_request_deny_on_the_allow_final_response_path() -> None: + await _assert_permission_deny_on_allow_final_response_path(step_count_is(1)) + + +async def test_fails_closed_on_schema_invalid_arguments_that_no_hook_repairs() -> None: + execute = Recorder({"ok": True}) + strict = tool( + name="strict", + input_schema=TargetArgs, + output_schema=OkOutput, + require_approval=lambda params, ctx: len(params["target"]) > 3, + execute=execute, + ) + client = QueuedClient( + [ + make_response("resp_turn_0", [function_call_item("call_strict", "strict", "{}")]), + _done("resp_final", "Recovered."), + ] + ) + accessor = MemoryStateAccessor() + + await call_model( + client, {"model": "test-model", "input": "do the strict thing", "tools": [strict], "state": accessor} + ).get_response() + + saved = accessor.stored + assert saved is not None + assert saved.status == "awaiting_approval" + assert saved.pending_tool_calls == [ParsedToolCall(id="call_strict", name="strict", arguments={})] + assert execute.count == 0 + assert len(client.requests) == 1 + + +def _danger_tool(execute: Recorder, input_schema: Any = TargetArgs) -> Dict[str, Any]: + return tool( + name="danger", input_schema=input_schema, output_schema=OkOutput, require_approval=True, execute=execute + ) + + +async def test_gates_duplicate_identical_calls_separately_but_only_once_across_visits() -> None: + danger_execute = Recorder({"ok": True}) + hooks = HooksManager() + permission_handler = Recorder({"decision": "allow"}) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_handler)) + client = QueuedClient( + [ + make_response( + "resp_duplicates", + [ + function_call_item("call_duplicate", "danger", '{"target":"prod"}'), + function_call_item("call_duplicate", "danger", '{"target":"prod"}'), + ], + ), + _done("resp_final"), + ] + ) + + await call_model( + client, + { + "model": "test-model", + "input": "do both things", + "tools": [_danger_tool(danger_execute)], + "hooks": hooks, + "state": MemoryStateAccessor(), + "stop_when": step_count_is(0), + "allow_final_response": True, + }, + ).get_text() + + # Two occurrences -> two gates, despite the pre-loop and the + # allow_final_response gate both visiting the response. + assert permission_handler.count == 2 + assert danger_execute.count == 2 + + +async def test_reuses_occurrence_identities_for_uncanonicalizable_response_calls() -> None: + # Upstream calls the private handleApprovalCheck three times (same response + # twice, then a later response). Driven through call_model instead: the + # first response (two deep-nested duplicates) is visited by both the + # pre-loop and in-loop gates; the later response carries one more + # occurrence. Each occurrence is gated exactly once -> 3 emits. + class ChildArgs(BaseModel): + child: Any = None + + danger_execute = Recorder({"ok": True}) + hooks = HooksManager() + permission_handler = Recorder({"decision": "allow"}) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_handler)) + deep = json.dumps(_deep_nested()) + client = QueuedClient( + [ + make_response( + "resp_deep_duplicates", + [function_call_item("call_deep", "danger", deep), function_call_item("call_deep", "danger", deep)], + ), + make_response("resp_deep_later", [function_call_item("call_deep", "danger", deep)]), + _done("resp_final"), + ] + ) + + await call_model( + client, + { + "model": "test-model", + "input": "do the thing", + "tools": [_danger_tool(danger_execute, ChildArgs)], + "hooks": hooks, + }, + ).get_response() + + assert permission_handler.count == 3 + assert danger_execute.count == 3 + assert len(client.requests) == 3 + + +@pytest.mark.skip( + reason="Private-API-only upstream case: it calls handleApprovalCheck with an empty `calls` subset " + "and relies on it deriving occurrences from response.output. The port's _handle_approval_check " + "gates exactly the calls passed (model_result.py:874-905) and every engine call site passes the " + "full extracted list, so the subset contract has no publicly observable counterpart." +) +async def test_derives_duplicate_occurrences_from_full_response_when_passed_a_subset() -> None: + """Upstream: growing response.output between two gate visits emits only for the new occurrence.""" + + +async def test_gates_initial_response_only_once_when_stop_fires_on_first_iteration() -> None: + danger_execute = Recorder({"ok": True}) + hooks = HooksManager() + permission_handler = Recorder({"decision": "allow"}) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_handler)) + client = QueuedClient( + [ + make_response("resp_turn_0", [function_call_item("call_danger", "danger", '{"target":"prod"}')]), + _done("resp_final"), + ] + ) + + text = await call_model( + client, + { + "model": "test-model", + "input": "do the thing", + "tools": [_danger_tool(danger_execute)], + "hooks": hooks, + "state": MemoryStateAccessor(), + "stop_when": step_count_is(0), + "allow_final_response": True, + }, + ).get_text() + + # One PermissionRequest emit for the single gated call -- not one per gate + # visit -- and the allowed call executes exactly once. + assert permission_handler.count == 1 + assert danger_execute.count == 1 + assert text == "Done." + assert len(client.requests) == 2 + + +# --------------------------------------------------------------------------- +# Approval decision validation and paused resumes (#517, #518). +# --------------------------------------------------------------------------- + + +class VArgs(BaseModel): + v: str + + +class AnswerOutput(BaseModel): + answer: str + + +def _caller_output(call_id: str, output: Any) -> Dict[str, Any]: + return { + "type": "function_call_output", + "id": f"output_{call_id}", + "callId": call_id, + "output": json.dumps(output, separators=(",", ":")), + } + + +def _gated_request(tools: Sequence[Tool], accessor: MemoryStateAccessor, **extra: Any) -> Dict[str, Any]: + return {"model": "test-model", "input": "run the gated action", "tools": list(tools), "state": accessor, **extra} + + +async def _pause_on_gated_calls(call_ids: Sequence[str]) -> Tuple[Recorder, List[Tool], MemoryStateAccessor]: + execute = Recorder({"ok": True}) + gated = tool(name="gated", input_schema=VArgs, output_schema=OkOutput, require_approval=True, execute=execute) + tools: List[Tool] = [gated] + client = QueuedClient( + [make_response("resp_pause", [function_call_item(call_id, "gated", '{"v":"x"}') for call_id in call_ids])] + ) + accessor = MemoryStateAccessor() + await call_model(client, _gated_request(tools, accessor)).get_response() + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + assert execute.count == 0 + return execute, tools, accessor + + +async def _pause_on_hitl_call() -> Tuple[Recorder, Recorder, List[Tool], MemoryStateAccessor]: + on_tool_called = Recorder(None) + on_response_received = Recorder() + + def transform(raw: Any, ctx: Any) -> Dict[str, Any]: + on_response_received(raw, ctx) + return {"answer": str(raw.get("answer")).upper()} + + review = tool( + name="gated", + input_schema=VArgs, + output_schema=AnswerOutput, + on_tool_called=on_tool_called, + on_response_received=transform, + ) + tools: List[Tool] = [review] + client = QueuedClient([make_response("resp_hitl", [function_call_item("call_hitl", "gated", '{"v":"x"}')])]) + accessor = MemoryStateAccessor() + await call_model(client, _gated_request(tools, accessor)).get_response() + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_hitl" + assert on_tool_called.count == 1 + return on_tool_called, on_response_received, tools, accessor + + +async def test_executes_an_approved_call_once_when_its_id_is_listed_twice() -> None: + execute, tools, accessor = await _pause_on_gated_calls(["call_1"]) + client = QueuedClient([_done("resp_done")]) + + text = await call_model(client, _gated_request(tools, accessor, approve_tool_calls=["call_1", "call_1"])).get_text() + + assert text == "Done." + assert execute.count == 1 + # Upstream asserts `callId`. Outgoing items use snake_case `call_id`: + # ModelResult._send normalizes at the transport boundary + # (model_result.py:213-231). Deliberate -- do not change back to callId. + assert [item["call_id"] for item in _function_call_outputs(client, 0)] == ["call_1"] + + +async def test_records_a_rejection_once_when_its_id_is_listed_twice() -> None: + execute, tools, accessor = await _pause_on_gated_calls(["call_1"]) + client = QueuedClient([_done("resp_done")]) + + await call_model(client, _gated_request(tools, accessor, reject_tool_calls=["call_1", "call_1"])).get_response() + + assert execute.count == 0 + outputs = _function_call_outputs(client, 0) + # snake_case `call_id` on the wire: see model_result.py:213-231 (_send). + assert [item["call_id"] for item in outputs] == ["call_1"] + assert json.loads(outputs[0]["output"]) == {"error": "Rejected by user"} + + +async def test_refuses_an_id_that_is_both_approved_and_rejected_before_anything_runs() -> None: + execute, tools, accessor = await _pause_on_gated_calls(["call_1", "call_2"]) + before = accessor.stored + saves_before = len(accessor.saved) + client = QueuedClient([]) + + with pytest.raises(ValueError, match="both approved and rejected: call_2"): + call_model( + client, + _gated_request(tools, accessor, approve_tool_calls=["call_1", "call_2"], reject_tool_calls=["call_2"]), + ) + + assert execute.count == 0 + assert client.requests == [] + assert accessor.stored is before + assert len(accessor.saved) == saves_before + + +async def test_stays_awaiting_approval_and_sends_no_request_when_resumed_without_decisions() -> None: + execute, tools, accessor = await _pause_on_gated_calls(["call_1"]) + before = accessor.stored + saves_before = len(accessor.saved) + client = QueuedClient([_done("resp_should_not_be_requested")]) + + result = call_model(client, _gated_request(tools, accessor)) + + with pytest.raises(RuntimeError, match=r"awaiting approval.*call_1"): + await result.get_response() + with pytest.raises(RuntimeError, match="awaiting approval"): + await result.get_text() + assert await result.requires_approval() is True + assert [call.id for call in await result.get_pending_tool_calls()] == ["call_1"] + + assert client.requests == [] + assert execute.count == 0 + assert accessor.stored is before + assert len(accessor.saved) == saves_before + + # The pause still resolves normally once a decision is supplied. + resume = QueuedClient([_done("resp_done")]) + await call_model(resume, _gated_request(tools, accessor, approve_tool_calls=["call_1"])).get_response() + assert execute.count == 1 + assert accessor.stored is not None + assert accessor.stored.status == "complete" + assert len(resume.requests) == 1 + + +async def test_stays_awaiting_hitl_and_sends_no_request_when_resumed_without_decisions() -> None: + on_tool_called = Recorder(None) + review = tool(name="gated", input_schema=VArgs, output_schema=OkOutput, on_tool_called=on_tool_called) + tools: List[Tool] = [review] + accessor = MemoryStateAccessor() + first = QueuedClient([make_response("resp_hitl", [function_call_item("call_hitl", "gated", '{"v":"x"}')])]) + await call_model(first, _gated_request(tools, accessor)).get_response() + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_hitl" + + client = QueuedClient([_done("resp_should_not_be_requested")]) + result = call_model(client, _gated_request(tools, accessor)) + + with pytest.raises(RuntimeError, match=r"awaiting HITL input.*call_hitl"): + await result.get_response() + assert client.requests == [] + assert on_tool_called.count == 1 + assert accessor.stored.status == "awaiting_hitl" + + +async def test_reports_the_pause_when_decisions_cover_only_some_pending_calls() -> None: + execute, tools, accessor = await _pause_on_gated_calls(["call_1", "call_2"]) + client = QueuedClient([]) + + result = call_model(client, _gated_request(tools, accessor, approve_tool_calls=["call_1"])) + + with pytest.raises(RuntimeError, match=r"awaiting approval.*call_2"): + await result.get_response() + assert execute.count == 1 + assert client.requests == [] + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + assert [call.id for call in accessor.stored.pending_tool_calls or []] == ["call_2"] + + +async def test_resumes_awaiting_hitl_when_input_carries_output_for_the_paused_call() -> None: + on_tool_called, on_response_received, tools, accessor = await _pause_on_hitl_call() + client = QueuedClient([_done("resp_done")]) + + text = await call_model( + client, + {**_gated_request(tools, accessor), "input": [_caller_output("call_hitl", {"answer": "yes"})]}, + ).get_text() + + assert text == "Done." + assert on_response_received.count == 1 + assert on_response_received.calls[0][0] == {"answer": "yes"} + # The paused call was not re-run; the model got the hook-transformed output. + assert on_tool_called.count == 1 + assert len(client.requests) == 1 + outputs = _function_call_outputs(client, 0) + # snake_case `call_id` on the wire: see model_result.py:213-231 (_send). + assert [(item["call_id"], item["output"]) for item in outputs] == [("call_hitl", '{"answer":"YES"}')] + assert accessor.stored is not None + assert accessor.stored.status == "complete" + + +async def test_resolves_a_function_valued_input_once_when_it_resumes_awaiting_hitl() -> None: + _, on_response_received, tools, accessor = await _pause_on_hitl_call() + client = QueuedClient([_done("resp_done")]) + input_fn = Recorder([_caller_output("call_hitl", {"answer": "yes"})]) + + text = await call_model(client, {**_gated_request(tools, accessor), "input": input_fn}).get_text() + + assert text == "Done." + assert input_fn.count == 1 + assert on_response_received.count == 1 + assert len(client.requests) == 1 + + +@pytest.mark.parametrize( + "input_value", + ["a new user message", [_caller_output("call_other", {"answer": "yes"})]], + ids=["string", "output-for-other-call"], +) +async def test_stays_awaiting_hitl_when_input_has_no_output_for_a_pending_call(input_value: Any) -> None: + _, on_response_received, tools, accessor = await _pause_on_hitl_call() + before = accessor.stored + saves_before = len(accessor.saved) + client = QueuedClient([]) + + result = call_model(client, {**_gated_request(tools, accessor), "input": input_value}) + + with pytest.raises( + RuntimeError, + match=r"awaiting HITL input.*call_hitl.*function_call_output.*input supplied with this call was not sent", + ): + await result.get_response() + assert client.requests == [] + assert on_response_received.count == 0 + assert accessor.stored is before + assert len(accessor.saved) == saves_before + + +async def test_stays_awaiting_approval_when_input_carries_output_for_the_gated_call() -> None: + execute, tools, accessor = await _pause_on_gated_calls(["call_1"]) + before = accessor.stored + saves_before = len(accessor.saved) + client = QueuedClient([]) + + result = call_model(client, {**_gated_request(tools, accessor), "input": [_caller_output("call_1", {"ok": True})]}) + + with pytest.raises(RuntimeError, match=r"awaiting approval.*call_1.*input supplied with this call was not sent"): + await result.get_response() + assert client.requests == [] + assert execute.count == 0 + assert accessor.stored is before + assert len(accessor.saved) == saves_before + + +async def test_rejects_every_response_and_stream_getter_on_a_paused_resume() -> None: + _, tools, accessor = await _pause_on_gated_calls(["call_1"]) + before = accessor.stored + saves_before = len(accessor.saved) + client = QueuedClient([]) + result = call_model(client, _gated_request(tools, accessor)) + + consumers: Dict[str, Callable[[], Awaitable[Any]]] = { + "get_response": result.get_response, + "get_text": result.get_text, + "get_tool_calls": result.get_tool_calls, + "get_text_stream": lambda: result.get_text_stream().__anext__(), + "get_reasoning_stream": lambda: result.get_reasoning_stream().__anext__(), + "get_full_responses_stream": lambda: result.get_full_responses_stream().__anext__(), + "get_items_stream": lambda: result.get_items_stream().__anext__(), + "get_new_messages_stream": lambda: result.get_new_messages_stream().__anext__(), + "get_tool_stream": lambda: result.get_tool_stream().__anext__(), + "get_tool_calls_stream": lambda: result.get_tool_calls_stream().__anext__(), + } + failures: List[str] = [] + for name, consume in consumers.items(): + error: Optional[BaseException] = None + try: + await consume() + except Exception as caught: # noqa: BLE001 - asserting the type below + error = caught + if not (isinstance(error, RuntimeError) and re.search(r"awaiting approval.*call_1", str(error))): + failures.append(f"{name}: {error!r}") + assert failures == [] + + # Inspection APIs still work on the paused conversation. + assert await result.requires_approval() is True + assert [call.id for call in await result.get_pending_tool_calls()] == ["call_1"] + assert (await result.get_state()).status == "awaiting_approval" + + assert client.requests == [] + assert accessor.stored is before + assert len(accessor.saved) == saves_before + + +@pytest.mark.parametrize( + ("status", "pending_tool_calls"), + [("awaiting_approval", []), ("awaiting_approval", None), ("awaiting_hitl", [])], + ids=["awaiting_approval-empty", "awaiting_approval-none", "awaiting_hitl-empty"], +) +async def test_continues_a_paused_state_with_nothing_pending( + status: str, pending_tool_calls: Optional[List[ParsedToolCall]] +) -> None: + # A paused status with nothing pending has no decision to wait for; + # pausing would strand the conversation, so it continues normally. + execute, tools, paused = await _pause_on_gated_calls(["call_1"]) + assert paused.stored is not None + accessor = MemoryStateAccessor() + accessor.stored = dataclasses.replace(paused.stored, status=status, pending_tool_calls=pending_tool_calls) + client = QueuedClient([_done("resp_done")]) + + text = await call_model(client, _gated_request(tools, accessor)).get_text() + + assert text == "Done." + assert len(client.requests) == 1 + assert execute.count == 0 + assert accessor.stored is not None + assert accessor.stored.status == "complete" + + +async def test_caps_pending_ids_in_awaiting_error_and_omits_input_note_without_input() -> None: + call_ids = [f"call_{index + 1}" for index in range(25)] + _, tools, accessor = await _pause_on_gated_calls(call_ids) + client = QueuedClient([]) + + with pytest.raises(RuntimeError) as caught: + await call_model(client, {**_gated_request(tools, accessor), "input": []}).get_response() + + message = str(caught.value) + assert "call_1, call_2," in message + assert "call_10, ... (15 more)" in message + assert "call_11" not in message + assert "was not sent" not in message + assert client.requests == [] + + +# Before de-duplication, the second copy of a duplicated ID skipped the +# re-approval that a PreToolUse argument mutation requires: the first copy +# re-gated the call (marking it pre_tool_use_applied), the second executed it. +async def test_does_not_let_a_duplicated_id_bypass_reapproval_after_pre_tool_use_mutation() -> None: + execute = Recorder({"ok": True}) + gated = tool( + name="gated", input_schema=VArgs, output_schema=OkOutput, require_approval=lambda p, c: True, execute=execute + ) + tools: List[Tool] = [gated] + accessor = MemoryStateAccessor() + first = QueuedClient([make_response("resp_pause", [function_call_item("call_1", "gated", '{"v":"reviewed"}')])]) + await call_model(first, _gated_request(tools, accessor)).get_response() + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=lambda p, c: {"mutated_input": {"v": "mutated"}})) + client = QueuedClient([]) + result = call_model(client, _gated_request(tools, accessor, hooks=hooks, approve_tool_calls=["call_1", "call_1"])) + + # The human approved {"v":"reviewed"}; the mutated arguments need their + # own approval, so nothing executes and the call is pending again. + with pytest.raises(RuntimeError, match=r"awaiting approval.*call_1"): + await result.get_response() + assert execute.count == 0 + assert client.requests == [] + assert accessor.stored.status == "awaiting_approval" + assert accessor.stored.pending_tool_calls == [ + ParsedToolCall(id="call_1", name="gated", arguments={"v": "mutated"}, pre_tool_use_applied=True) + ] + + +# --------------------------------------------------------------------------- +# Port-added coverage for the two-phase PreToolUse preparation (upstream +# c610b6e). Upstream ships no dedicated cases for these branches of +# prepareAfterInitialApproval / processApprovalDecisions; each test pins one. +# --------------------------------------------------------------------------- + + +def _mutating_hooks(mutated: Dict[str, Any]) -> Tuple[HooksManager, Recorder, Recorder]: + pre_tool_use = Recorder({"mutated_input": mutated}) + failures = Recorder() + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=pre_tool_use)) + hooks.on(HookName.PostToolUseFailure.value, HookEntry(handler=failures)) + return hooks, pre_tool_use, failures + + +async def _pause_on_mutated_gate(hooks: HooksManager, execute: Recorder) -> Tuple[List[Tool], MemoryStateAccessor]: + def require(params: Any, ctx: Any) -> bool: + return bool(params["v"] == "mutated") + + gated = tool(name="gated", input_schema=VArgs, output_schema=OkOutput, require_approval=require, execute=execute) + tools: List[Tool] = [gated] + accessor = MemoryStateAccessor() + client = QueuedClient([make_response("resp_pause", [function_call_item("call_1", "gated", '{"v":"wire"}')])]) + await call_model(client, _gated_request(tools, accessor, hooks=hooks)).get_response() + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + assert accessor.stored.pending_tool_calls == [ + ParsedToolCall(id="call_1", name="gated", arguments={"v": "mutated"}, pre_tool_use_applied=True) + ] + return tools, accessor + + +async def test_approving_a_prepared_call_executes_its_effective_input_without_rerunning_pre_tool_use() -> None: + execute = Recorder({"ok": True}) + hooks, pre_tool_use, failures = _mutating_hooks({"v": "mutated"}) + tools, accessor = await _pause_on_mutated_gate(hooks, execute) + assert pre_tool_use.count == 1 + + client = QueuedClient([_done("resp_done")]) + text = await call_model( + client, _gated_request(tools, accessor, hooks=hooks, approve_tool_calls=["call_1"]) + ).get_text() + + assert text == "Done." + # The marker means PreToolUse already ran: it does not run a second time. + assert pre_tool_use.count == 1 + assert execute.count == 1 + assert _dumped_first_args(execute) == [{"v": "mutated"}] + assert failures.count == 0 + assert len(client.requests) == 1 + + +async def test_rejecting_a_prepared_call_emits_post_tool_use_failure() -> None: + execute = Recorder({"ok": True}) + hooks, pre_tool_use, failures = _mutating_hooks({"v": "mutated"}) + tools, accessor = await _pause_on_mutated_gate(hooks, execute) + + client = QueuedClient([_done("resp_done")]) + await call_model(client, _gated_request(tools, accessor, hooks=hooks, reject_tool_calls=["call_1"])).get_response() + + assert execute.count == 0 + assert pre_tool_use.count == 1 + # The call's lifecycle entered PreToolUse, so the user rejection is a + # lifecycle failure (hooks_schemas.PostToolUseFailurePayload). + assert [(call[0]["tool_input"], call[0]["error"]) for call in failures.calls] == [ + ({"v": "mutated"}, "Rejected by user") + ] + outputs = _function_call_outputs(client, 0) + assert [(item["call_id"], json.loads(item["output"])) for item in outputs] == [ + ("call_1", {"error": "Rejected by user"}) + ] + + +async def test_rejecting_an_unprepared_call_does_not_emit_post_tool_use_failure() -> None: + execute, tools, accessor = await _pause_on_gated_calls(["call_1"]) + hooks, pre_tool_use, failures = _mutating_hooks({"v": "mutated"}) + + client = QueuedClient([_done("resp_done")]) + await call_model(client, _gated_request(tools, accessor, hooks=hooks, reject_tool_calls=["call_1"])).get_response() + + # Initial approval rejections never enter the tool lifecycle. + assert pre_tool_use.count == 0 + assert failures.count == 0 + assert execute.count == 0 + + +async def test_blocks_a_schema_invalid_pre_tool_use_mutation() -> None: + execute = Recorder({"ok": True}) + permission_request = Recorder({"decision": "allow"}) + hooks, pre_tool_use, failures = _mutating_hooks({"v": 42}) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_request)) + gated = tool( + name="gated", input_schema=VArgs, output_schema=OkOutput, require_approval=lambda p, c: False, execute=execute + ) + client = QueuedClient( + [make_response("resp_invalid", [function_call_item("call_1", "gated", '{"v":"wire"}')]), _done("resp_done")] + ) + + text = await call_model( + client, {"model": "test-model", "input": "run", "tools": [gated], "hooks": hooks} + ).get_text() + + reason = 'PreToolUse produced invalid input for "gated"' + assert text == "Done." + assert execute.count == 0 + assert pre_tool_use.count == 1 + # Never re-gated: the invalid input is blocked before approval. + assert permission_request.count == 0 + assert [(call[0]["tool_input"], call[0]["error"]) for call in failures.calls] == [({"v": 42}, reason)] + outputs = _function_call_outputs(client, 1) + assert [(item["call_id"], json.loads(item["output"])) for item in outputs] == [("call_1", {"error": reason})] + + +async def test_permission_request_deny_on_the_mutated_gate_blocks_without_pausing() -> None: + execute = Recorder({"ok": True}) + permission_request = Recorder({"decision": "deny", "reason": "mutation not allowed"}) + hooks, pre_tool_use, failures = _mutating_hooks({"v": "mutated"}) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_request)) + + def require(params: Any, ctx: Any) -> bool: + return bool(params["v"] == "mutated") + + gated = tool(name="gated", input_schema=VArgs, output_schema=OkOutput, require_approval=require, execute=execute) + accessor = MemoryStateAccessor() + client = QueuedClient( + [make_response("resp_deny", [function_call_item("call_1", "gated", '{"v":"wire"}')]), _done("resp_done")] + ) + + text = await call_model(client, _gated_request([gated], accessor, hooks=hooks)).get_text() + + assert text == "Done." + assert execute.count == 0 + # Only the mutated input needed approval, so one PermissionRequest, on it. + assert [call[0]["tool_input"] for call in permission_request.calls] == [{"v": "mutated"}] + assert [(call[0]["tool_input"], call[0]["error"]) for call in failures.calls] == [ + ({"v": "mutated"}, "mutation not allowed") + ] + outputs = _function_call_outputs(client, 1) + assert [(item["call_id"], json.loads(item["output"])) for item in outputs] == [ + ("call_1", {"error": "mutation not allowed"}) + ] + assert accessor.stored is not None + assert accessor.stored.status == "complete" + + +async def test_call_level_require_approval_re_gates_mutated_input() -> None: + execute = Recorder({"ok": True}) + hooks, pre_tool_use, _ = _mutating_hooks({"v": "mutated"}) + call_level = Recorder() + + def require(call: ParsedToolCall, ctx: Any) -> bool: + call_level(call, ctx) + return bool(call.arguments["v"] == "mutated") + + # No tool-level require_approval: only the call-level source can depend + # on input, and it alone re-gates the mutation. + plain = tool(name="gated", input_schema=VArgs, output_schema=OkOutput, execute=execute) + accessor = MemoryStateAccessor() + client = QueuedClient([make_response("resp_call_level", [function_call_item("call_1", "gated", '{"v":"wire"}')])]) + + await call_model(client, _gated_request([plain], accessor, hooks=hooks, require_approval=require)).get_response() + + assert [call[0].arguments for call in call_level.calls] == [{"v": "wire"}, {"v": "mutated"}] + assert pre_tool_use.count == 1 + assert execute.count == 0 + assert accessor.stored is not None + assert accessor.stored.pending_tool_calls == [ + ParsedToolCall(id="call_1", name="gated", arguments={"v": "mutated"}, pre_tool_use_applied=True) + ] + + +async def test_static_require_approval_true_does_not_re_gate_mutated_input() -> None: + execute = Recorder({"ok": True}) + permission_request = Recorder({"decision": "allow"}) + hooks, pre_tool_use, _ = _mutating_hooks({"v": "mutated"}) + hooks.on(HookName.PermissionRequest.value, HookEntry(handler=permission_request)) + gated = tool(name="gated", input_schema=VArgs, output_schema=OkOutput, require_approval=True, execute=execute) + client = QueuedClient( + [make_response("resp_static", [function_call_item("call_1", "gated", '{"v":"wire"}')]), _done("resp_done")] + ) + + await call_model(client, {"model": "test-model", "input": "run", "tools": [gated], "hooks": hooks}).get_text() + + # A blanket `True` cannot depend on input: one PermissionRequest (on the + # wire input), then PreToolUse exactly once, then execution of the + # mutated input exactly once. + assert [call[0]["tool_input"] for call in permission_request.calls] == [{"v": "wire"}] + assert pre_tool_use.count == 1 + assert execute.count == 1 + assert _dumped_first_args(execute) == [{"v": "mutated"}] + + +async def test_pre_tool_use_block_during_preparation_skips_execution_once() -> None: + execute = Recorder({"ok": True}) + pre_tool_use = Recorder({"block": "nope"}) + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=pre_tool_use)) + safe = tool(name="safe", input_schema=VArgs, output_schema=OkOutput, execute=execute) + client = QueuedClient( + [make_response("resp_block", [function_call_item("call_1", "safe", '{"v":"x"}')]), _done("resp_done")] + ) + + await call_model(client, {"model": "test-model", "input": "run", "tools": [safe], "hooks": hooks}).get_text() + + assert pre_tool_use.count == 1 + assert execute.count == 0 + outputs = _function_call_outputs(client, 1) + assert [(item["call_id"], json.loads(item["output"])) for item in outputs] == [("call_1", {"error": "nope"})] + + +async def test_pre_tool_use_runs_exactly_once_per_call_across_a_mixed_pause_and_resume() -> None: + safe_execute, danger_execute, tools = _safe_and_danger_tools() + pre_tool_use = Recorder() + hooks = HooksManager() + hooks.on(HookName.PreToolUse.value, HookEntry(handler=pre_tool_use)) + accessor = MemoryStateAccessor() + first = QueuedClient( + [ + make_response( + "resp_mixed", + [ + function_call_item("call_safe", "safe", '{"target":"staging"}'), + function_call_item("call_danger", "danger", '{"target":"prod"}'), + ], + ) + ] + ) + await call_model(first, _gated_request(tools, accessor, hooks=hooks)).get_response() + + # The auto-approved sibling was prepared and executed while pausing; the + # gated call is still pending on its initial approval, so its PreToolUse + # has not run and it carries no marker. + assert [call[0]["tool_name"] for call in pre_tool_use.calls] == ["safe"] + assert safe_execute.count == 1 + assert danger_execute.count == 0 + assert accessor.stored is not None + assert accessor.stored.pending_tool_calls == [ + ParsedToolCall(id="call_danger", name="danger", arguments={"target": "prod"}) + ] + + resume = QueuedClient([_done("resp_done")]) + await call_model( + resume, _gated_request(tools, accessor, hooks=hooks, approve_tool_calls=["call_danger"]) + ).get_text() + + assert [call[0]["tool_name"] for call in pre_tool_use.calls] == ["safe", "danger"] + assert safe_execute.count == 1 + assert danger_execute.count == 1 + assert [item["call_id"] for item in _function_call_outputs(resume, 0)] == ["call_safe", "call_danger"] diff --git a/tests/unit/test_call_model.py b/tests/unit/test_call_model.py index 83be11a..9b96fb8 100644 --- a/tests/unit/test_call_model.py +++ b/tests/unit/test_call_model.py @@ -63,7 +63,10 @@ async def test_allow_final_response_executes_pending_tool_before_no_tools_turn() "model": "test/model", "input": "double 2", "tools": [double], - "stop_when": step_count_is(1), + # Stop conditions see executed tool rounds (upstream + # shouldStopExecution), so step_count_is(0) halts on the first + # response's calls -- upstream allow-final-response.test.ts. + "stop_when": step_count_is(0), "allow_final_response": "summarize", }, ) diff --git a/tests/unit/test_call_model_active_tools.py b/tests/unit/test_call_model_active_tools.py new file mode 100644 index 0000000..860276e --- /dev/null +++ b/tests/unit/test_call_model_active_tools.py @@ -0,0 +1,183 @@ +"""Ports `packages/agent/tests/unit/call-model-active-tools.test.ts`. + +Upstream semantics (`src/inner-loop/call-model.ts:122-149`): `activeTools` +narrows `tools` *before* API conversion and before the tools are handed to +`ModelResult` for execution. Server tools always survive the filter (only +client `function.name`s are matched); unknown names are ignored; a list that +filters to empty -- or an explicitly empty `tools` -- omits the outbound +`tools` key entirely. `activeTools` itself is client-only and never sent. + +Divergence from upstream's harness: upstream installs a capturing +`HTTPClient` and aborts the request; this port drives `call_model` through +`tests._fixtures.QueuedClient` and asserts on the recorded `send_async` +kwargs, which is the transport boundary this port binds to. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +import pytest + +from openrouter_agent import call_model +from openrouter_agent.tool import server_tool, tool +from openrouter_agent.tool_set import TOOL_SET_SNAPSHOT, create_tool_set, strip_tool_set_snapshot_metadata +from tests._fixtures import MemoryStateAccessor, QueuedClient, text_response, tool_call_response + + +class ExecutionCounter: + def __init__(self) -> None: + self.count = 0 + + def __call__(self, params: Any, ctx: Any) -> Dict[str, Any]: + self.count += 1 + return {"ok": True} + + +def make_tool(name: str, execute: Any = None) -> Dict[str, Any]: + return tool(name=name, input_schema=dict, execute=execute or ExecutionCounter()) + + +tool_a = make_tool("a") +tool_b = make_tool("b") + + +def tool_names(request: Dict[str, Any]) -> List[str]: + return [t["name"] for t in request.get("tools") or [] if isinstance(t, dict) and isinstance(t.get("name"), str)] + + +async def capture_request(request: Dict[Any, Any]) -> Dict[str, Any]: + """Run `call_model` to completion against one queued text turn and return + the single request it sent.""" + client = QueuedClient([text_response("r1", "ok")]) + await call_model(client, {"model": "openai/gpt-4o-mini", "input": "hi", **request}).get_text() + assert len(client.requests) == 1 + return client.requests[0] + + +class TestActiveToolsFilter: + async def test_sends_only_active_tools_when_active_tools_is_provided(self) -> None: + sent = await capture_request({"tools": [tool_a, tool_b], "active_tools": ["a"]}) + assert tool_names(sent) == ["a"] + # Client-only field (async-params.ts clientOnlyFields): never on the wire. + assert "active_tools" not in sent + assert "activeTools" not in sent + + async def test_silently_ignores_unknown_active_tools_names(self) -> None: + sent = await capture_request({"tools": [tool_a, tool_b], "active_tools": ["a", "missing"]}) + assert tool_names(sent) == ["a"] + + async def test_sends_all_tools_when_active_tools_is_omitted(self) -> None: + sent = await capture_request({"tools": [tool_a, tool_b]}) + assert tool_names(sent) == ["a", "b"] + + @pytest.mark.skip( + reason="this port has no long-running (lifecycle='background') tools or universal `task` tool to filter" + ) + async def test_does_not_advertise_the_task_helper_when_its_background_tool_is_filtered_out(self) -> None: + raise AssertionError("unreachable") + + async def test_omits_the_tools_key_when_tools_is_explicitly_empty(self) -> None: + sent = await capture_request({"tools": []}) + assert "tools" not in sent + + async def test_omits_the_tools_key_when_active_tools_filters_out_every_tool(self) -> None: + # Several providers reject an explicit empty tools array, so the key + # must be absent rather than `[]`. + sent = await capture_request({"tools": [tool_a, tool_b], "active_tools": ["missing"]}) + assert "tools" not in sent + + async def test_empty_active_tools_list_filters_every_client_tool(self) -> None: + # Upstream: `activeTools ? new Set(activeTools) : undefined` -- an empty + # array is truthy in JS, so `[]` means "no client tools", unlike None. + sent = await capture_request({"tools": [tool_a, tool_b], "active_tools": []}) + assert "tools" not in sent + + async def test_server_tools_always_pass_the_active_tools_filter(self) -> None: + # Upstream filter: `isServerTool(t) || activeSet.has(t.function.name)`. + web_search = server_tool({"type": "web_search_2025_08_26"}) + sent = await capture_request({"tools": [tool_a, web_search, tool_b], "active_tools": ["b"]}) + assert [(t["type"], t.get("name")) for t in sent["tools"]] == [ + ("web_search_2025_08_26", None), + ("function", "b"), + ] + + async def test_filtered_tool_is_not_executable(self) -> None: + # Upstream registers only the filtered tools on ModelResult + # (call-model.ts `tools: filteredTools`), so a call to a filtered tool + # is an unknown name: not executed, no error output, the run stops and + # surfaces it as an unresolved client call (model-result.ts + # executeSingleToolCall -> null; persistClientToolsPause). + execute_b = ExecutionCounter() + b_counted = make_tool("b", execute_b) + state = MemoryStateAccessor() + client = QueuedClient([tool_call_response("r1", "b", call_id="call_b"), text_response("r2", "unused")]) + + await call_model( + client, + {"model": "m", "input": "hi", "tools": [tool_a, b_counted], "active_tools": ["a"], "state": state}, + ).get_response() + + assert execute_b.count == 0 + assert len(client.requests) == 1 + assert tool_names(client.requests[0]) == ["a"] + assert state.stored is not None + assert state.stored.status == "awaiting_client_tools" + assert [c.name for c in state.stored.pending_tool_calls or []] == ["b"] + + +class TestToolSetSnapshotStripping: + async def test_never_sends_snapshot_metadata_when_a_whole_snapshot_is_spread_in(self) -> None: + # Mirrors the documented-but-dangerous pattern of spreading the full + # `infer_tools()` result straight into call_model. + snapshot_like_request: Dict[Any, Any] = { + "tools": [tool_a], + "active_tools": ["a"], + "enabled": ["a"], + "disabled": [], + "status_by_tool": {"a": "enabled"}, + "call_model": {"tools": [tool_a], "active_tools": ["a"]}, + TOOL_SET_SNAPSHOT: True, + } + sent = await capture_request(snapshot_like_request) + for key in ("enabled", "disabled", "status_by_tool", "call_model", TOOL_SET_SNAPSHOT): + assert key not in sent + assert tool_names(sent) == ["a"] + + async def test_whole_infer_tools_result_can_be_spread(self) -> None: + snapshot = create_tool_set([tool_a, tool_b]).deactivate("b").infer_tools() + sent = await capture_request(snapshot) + assert tool_names(sent) == ["a"] + for key in ("enabled", "disabled", "status_by_tool", TOOL_SET_SNAPSHOT): + assert key not in sent + + def test_preserves_identically_named_fields_when_not_a_tool_set_snapshot(self) -> None: + request: Dict[Any, Any] = { + "enabled": True, + "disabled": False, + "status_by_tool": {"a": "request-value"}, + "call_model": "request-value", + } + strip_tool_set_snapshot_metadata(request) + assert request == { + "enabled": True, + "disabled": False, + "status_by_tool": {"a": "request-value"}, + "call_model": "request-value", + } + + def test_strips_metadata_and_marker_from_a_marked_snapshot(self) -> None: + request = create_tool_set([tool_a]).infer_tools() + strip_tool_set_snapshot_metadata(request) + assert request == {"tools": [tool_a], "active_tools": ["a"]} + + async def test_still_sends_the_documented_tools_active_tools_pattern(self) -> None: + sent = await capture_request({"tools": [tool_a], "active_tools": ["a"]}) + assert tool_names(sent) == ["a"] + assert "active_tools" not in sent + + async def test_resolved_call_model_mapping_spreads_into_call_model(self) -> None: + resolved = create_tool_set([tool_a, tool_b]).deactivate("b").resolve() + sent = await capture_request({**resolved.call_model}) + assert tool_names(sent) == ["a"] + assert "active_tools" not in sent diff --git a/tests/unit/test_chat_compat.py b/tests/unit/test_chat_compat.py new file mode 100644 index 0000000..dea9b2a --- /dev/null +++ b/tests/unit/test_chat_compat.py @@ -0,0 +1,191 @@ +"""Ports `packages/agent/src/lib/chat-compat.test.ts` (fromChatMessages cases, +including the assistant tool-call regression for upstream issue #11). + +Deliberate divergence, asserted throughout: this port's message items carry an +explicit ``"type": "message"`` (pre-existing shape of `chat_compat.py`, also +asserted in tests/unit/test_compat.py), where upstream emits bare +``{role, content}`` EasyInputMessages. Both are valid Responses input items. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +from openrouter_agent import from_chat_messages + + +def _message(role: str, content: str) -> Dict[str, Any]: + return {"type": "message", "role": role, "content": content} + + +def _function_call(call_id: str, name: str, arguments: str) -> Dict[str, Any]: + return { + "type": "function_call", + "callId": call_id, + "id": call_id, + "name": name, + "arguments": arguments, + "status": "completed", + } + + +def _tool_call(call_id: str, name: str, arguments: str) -> Dict[str, Any]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": arguments}} + + +# -- basic message conversion ---------------------------------------------------- + + +def test_converts_each_role_with_string_content() -> None: + for role in ("user", "assistant", "system", "developer"): + assert from_chat_messages([{"role": role, "content": "Hello"}]) == [_message(role, "Hello")] + + +def test_converts_multiple_messages_in_order() -> None: + result = from_chat_messages( + [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + ) + assert result == [ + _message("system", "You are helpful."), + _message("user", "Hi"), + _message("assistant", "Hello!"), + ] + + +# -- tool response conversion ----------------------------------------------------- + + +def test_converts_tool_message_to_function_call_output() -> None: + assert from_chat_messages([{"role": "tool", "content": "Sunny", "tool_call_id": "call_1"}]) == [ + {"type": "function_call_output", "callId": "call_1", "output": "Sunny"} + ] + + +def test_converts_tool_message_with_object_content_by_stringifying() -> None: + result = from_chat_messages([{"role": "tool", "content": {"temp": 22}, "toolCallId": "call_1"}]) + assert result == [{"type": "function_call_output", "callId": "call_1", "output": '{"temp":22}'}] + + +# -- null and empty content ------------------------------------------------------- + + +def test_null_assistant_content_without_tool_calls_round_trips_as_empty_message() -> None: + assert from_chat_messages([{"role": "assistant", "content": None}]) == [_message("assistant", "")] + + +def test_empty_string_content() -> None: + assert from_chat_messages([{"role": "user", "content": ""}]) == [_message("user", "")] + + +def test_empty_messages_list() -> None: + assert from_chat_messages([]) == [] + + +# -- assistant tool call conversion (#11) ----------------------------------------- + + +def test_emits_function_call_for_assistant_with_null_content_and_one_tool_call() -> None: + result = from_chat_messages( + [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [_tool_call("call_123", "get_weather", '{"location":"Paris"}')], + }, + {"role": "tool", "content": "Sunny, 22C", "tool_call_id": "call_123"}, + ] + ) + + assert result == [ + _message("user", "What is the weather in Paris?"), + _function_call("call_123", "get_weather", '{"location":"Paris"}'), + {"type": "function_call_output", "callId": "call_123", "output": "Sunny, 22C"}, + ] + + +def test_emits_message_then_function_call_when_assistant_has_text_and_tool_calls() -> None: + result = from_chat_messages( + [ + { + "role": "assistant", + "content": "Let me check the weather for you.", + "toolCalls": [_tool_call("call_456", "get_weather", '{"location":"London"}')], + } + ] + ) + + assert result == [ + _message("assistant", "Let me check the weather for you."), + _function_call("call_456", "get_weather", '{"location":"London"}'), + ] + + +def test_emits_one_function_call_per_parallel_tool_call_in_order() -> None: + result = from_chat_messages( + [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + _tool_call("call_a", "get_weather", '{"location":"Paris"}'), + _tool_call("call_b", "get_time", '{"tz":"UTC"}'), + ], + } + ] + ) + + assert result == [ + _function_call("call_a", "get_weather", '{"location":"Paris"}'), + _function_call("call_b", "get_time", '{"tz":"UTC"}'), + ] + + +def test_does_not_re_stringify_serialized_tool_call_arguments() -> None: + result = from_chat_messages( + [{"role": "assistant", "content": None, "tool_calls": [_tool_call("call_raw", "noop", '{"a":1}')]}] + ) + + # Would be '"{\\"a\\":1}"' if json.dumps were applied a second time. + assert len(result) == 1 + assert result[0]["arguments"] == '{"a":1}' + + +def test_mapping_arguments_are_serialized_once() -> None: + # Port-specific: a pre-parsed dict (not valid chat format, but common in + # Python callers) is serialized exactly once rather than forwarded raw. + result = from_chat_messages( + [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "c", "type": "function", "function": {"name": "n", "arguments": {"a": 1}}}], + } + ] + ) + assert result == [_function_call("c", "n", '{"a":1}')] + + +def test_empty_tool_calls_list_emits_only_the_message() -> None: + result = from_chat_messages([{"role": "assistant", "content": "No tools needed.", "tool_calls": []}]) + + assert result == [_message("assistant", "No tools needed.")] + + +def test_full_conversation_items_are_all_input_item_shaped() -> None: + # Upstream #41 is a compile-time assignability check (`Item[]`); the runtime + # analogue is that every emitted item is a Responses input item. + items: List[Dict[str, Any]] = from_chat_messages( + [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": None, "tool_calls": [_tool_call("call_typed", "get_weather", "{}")]}, + {"role": "tool", "content": "ok", "tool_call_id": "call_typed"}, + ] + ) + + assert [item["type"] for item in items] == ["message", "message", "function_call", "function_call_output"] diff --git a/tests/unit/test_claude_type_guards.py b/tests/unit/test_claude_type_guards.py new file mode 100644 index 0000000..a0574bf --- /dev/null +++ b/tests/unit/test_claude_type_guards.py @@ -0,0 +1,106 @@ +"""Ports `packages/agent/tests/unit/claude-type-guards.test.ts`.""" + +from __future__ import annotations + +from openrouter_agent import is_claude_style_messages +from openrouter_agent.claude_constants import ClaudeContentBlockType + +TOOL_RESULT = ClaudeContentBlockType.ToolResult.value +IMAGE = ClaudeContentBlockType.Image.value +TOOL_USE = ClaudeContentBlockType.ToolUse.value + + +def test_returns_false_for_non_array_input() -> None: + assert is_claude_style_messages("hello") is False + assert is_claude_style_messages(None) is False + assert is_claude_style_messages(42) is False + assert is_claude_style_messages({"role": "user"}) is False + + +def test_returns_false_for_an_empty_array() -> None: + assert is_claude_style_messages([]) is False + + +def test_detects_tool_result_content_blocks() -> None: + messages = [{"role": "user", "content": [{"type": TOOL_RESULT, "tool_use_id": "x", "content": "ok"}]}] + assert is_claude_style_messages(messages) is True + + +def test_detects_image_blocks_with_a_source() -> None: + messages = [ + { + "role": "user", + "content": [{"type": IMAGE, "source": {"type": "base64", "media_type": "image/png", "data": "abc"}}], + } + ] + assert is_claude_style_messages(messages) is True + + +def test_rejects_image_blocks_without_a_source_object() -> None: + assert is_claude_style_messages([{"role": "user", "content": [{"type": IMAGE}]}]) is False + assert is_claude_style_messages([{"role": "user", "content": [{"type": IMAGE, "source": "base64"}]}]) is False + + +def test_detects_tool_use_blocks_with_a_string_id() -> None: + messages = [{"role": "assistant", "content": [{"type": TOOL_USE, "id": "toolu_1", "name": "x", "input": {}}]}] + assert is_claude_style_messages(messages) is True + + +def test_rejects_tool_use_blocks_with_a_non_string_id() -> None: + assert is_claude_style_messages([{"role": "assistant", "content": [{"type": TOOL_USE, "id": 123}]}]) is False + + +def test_returns_false_for_plain_openai_style_string_content() -> None: + messages = [{"role": "user", "content": "hello"}, {"role": "assistant", "content": "hi there"}] + assert is_claude_style_messages(messages) is False + + +def test_returns_false_when_any_message_has_a_non_claude_role() -> None: + assert is_claude_style_messages([{"role": "system", "content": [{"type": TOOL_RESULT}]}]) is False + assert is_claude_style_messages([{"role": "developer", "content": [{"type": TOOL_RESULT}]}]) is False + assert is_claude_style_messages([{"role": "tool", "content": [{"type": IMAGE, "source": {}}]}]) is False + + +def test_non_claude_role_after_a_claude_block_does_not_override_detection() -> None: + # Upstream scans in order and returns at the first decisive message. + messages = [ + {"role": "user", "content": [{"type": TOOL_RESULT}]}, + {"role": "system", "content": "late"}, + ] + assert is_claude_style_messages(messages) is True + + +def test_skips_non_record_entries_and_messages_without_a_role() -> None: + messages = [ + "not-a-message", + {"content": [{"type": TOOL_RESULT}]}, # no role + {"role": "user", "content": [{"type": TOOL_RESULT}]}, + ] + assert is_claude_style_messages(messages) is True + + +def test_skips_messages_that_carry_a_top_level_type() -> None: + messages = [{"role": "user", "type": "message", "content": [{"type": TOOL_RESULT}]}] + assert is_claude_style_messages(messages) is False + + +def test_returns_false_when_content_blocks_are_all_plain_text() -> None: + assert is_claude_style_messages([{"role": "user", "content": [{"type": "text", "text": "hello"}]}]) is False + + +def test_ignores_non_record_blocks_inside_content_arrays() -> None: + assert is_claude_style_messages([{"role": "user", "content": ["just-a-string", None, 42]}]) is False + + +def test_detects_claude_blocks_in_later_messages_even_if_earlier_ones_are_plain() -> None: + messages = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": TOOL_USE, "id": "toolu_9", "name": "x", "input": {}}]}, + ] + assert is_claude_style_messages(messages) is True + + +def test_accepts_enum_members_as_block_types() -> None: + # ClaudeContentBlockType is a str Enum, so members compare equal to values. + messages = [{"role": "user", "content": [{"type": ClaudeContentBlockType.ToolResult}]}] + assert is_claude_style_messages(messages) is True diff --git a/tests/unit/test_conversation_state_serialization.py b/tests/unit/test_conversation_state_serialization.py index 74fe1b6..96a53c2 100644 --- a/tests/unit/test_conversation_state_serialization.py +++ b/tests/unit/test_conversation_state_serialization.py @@ -143,3 +143,45 @@ class FakeSDKItem(BaseModel): parsed = json.loads(raw) assert parsed["messages"][0]["callId"] == "call_1" + + +def test_round_trips_the_pre_tool_use_applied_marker_on_pending_calls() -> None: + """Upstream `PendingToolCall.preToolUseApplied` (tool-types.ts): a call + paused on the mutated-input approval gate persists its effective + arguments with the marker so the resume does not re-run PreToolUse. The + marker is written only when set, like upstream's `?: true`.""" + state = dataclasses.replace( + create_initial_state("conv_prepared"), + status="awaiting_approval", + pending_tool_calls=[ + ParsedToolCall(id="call_prepared", name="gated", arguments={"v": "mutated"}, pre_tool_use_applied=True), + ParsedToolCall(id="call_plain", name="gated", arguments={"v": "x"}), + ], + ) + + raw = serialize_conversation_state(state) + restored = deserialize_conversation_state(raw) + + assert restored == state + assert [item.get("pre_tool_use_applied") for item in json.loads(raw)["pending_tool_calls"]] == [True, None] + assert [call.pre_tool_use_applied for call in restored.pending_tool_calls or []] == [True, False] + + +def test_loads_legacy_pending_calls_without_the_pre_tool_use_applied_marker() -> None: + # A blob written before the marker existed: the call has not been through + # PreToolUse yet, so the approval resume must prepare it. + legacy_json = json.dumps( + { + "id": "conv_legacy_pending", + "messages": [], + "status": "awaiting_approval", + "created_at": 1, + "updated_at": 1, + "pending_tool_calls": [{"id": "call_legacy", "name": "gated", "arguments": {"v": "x"}}], + } + ) + + restored = deserialize_conversation_state(legacy_json) + + assert restored.pending_tool_calls == [ParsedToolCall(id="call_legacy", name="gated", arguments={"v": "x"})] + assert restored.pending_tool_calls[0].pre_tool_use_applied is False diff --git a/tests/unit/test_doom_loop.py b/tests/unit/test_doom_loop.py new file mode 100644 index 0000000..978ef8d --- /dev/null +++ b/tests/unit/test_doom_loop.py @@ -0,0 +1,926 @@ +"""Unit tests for the doom-loop detection primitives (port of tests/unit/doom-loop.test.ts). + +Everything here is pure and deterministic: JCS canonicalization, SHA-256 +fingerprints (asserted against the cross-port vector file), text-repetition +detection, ladder resolution, loop_key resolution, and the DoomLoopMonitor +state machine (round-scoped streaks). Loop integration (blocking, steering, +stopping a real run) is upstream's doom-loop-integration.test.ts, which needs +the engine wiring and is not part of this file. +""" + +from __future__ import annotations + +import json +import re +import warnings +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import pytest + +from openrouter_agent.doom_loop import ( + DEFAULT_DOOM_LOOP_LADDER, + MAX_CANONICALIZE_DEPTH, + DoomLoopCallRecord, + DoomLoopLadder, + DoomLoopMonitor, + DoomLoopOption, + DoomLoopVerdict, + canonicalize_key_material, + detect_text_repetition, + fingerprint_key_material, + fingerprint_tool_call, + resolve_doom_loop_option, + resolve_ladder_action, + resolve_loop_key_material, +) + +VECTORS_PATH = Path(__file__).resolve().parent.parent / "vectors" / "doom-loop-fingerprints.json" +VECTORS: Dict[str, Any] = json.loads(VECTORS_PATH.read_text(encoding="utf-8")) + + +def _subset(actual: Optional[Any], expected: Dict[str, Any]) -> None: + """`toMatchObject`: every expected key present with an equal value.""" + assert actual is not None + for key, value in expected.items(): + assert key in actual, key + assert actual[key] == value, (key, actual[key], value) + + +def _make_monitor(overrides: Optional[DoomLoopOption] = None) -> DoomLoopMonitor: + return DoomLoopMonitor(resolve_doom_loop_option(True if overrides is None else overrides)) # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Canonicalization (RFC 8785 / JCS semantics) +# --------------------------------------------------------------------------- + + +class TestCanonicalizeKeyMaterial: + def test_is_insensitive_to_object_key_order_recursively(self) -> None: + assert canonicalize_key_material({"b": 2, "a": {"d": 4, "c": 3}}) == canonicalize_key_material( + {"a": {"c": 3, "d": 4}, "b": 2} + ) + assert canonicalize_key_material({"b": 2, "a": {"d": 4, "c": 3}}) == '{"a":{"c":3,"d":4},"b":2}' + + def test_distinguishes_array_order(self) -> None: + assert canonicalize_key_material([1, 2]) != canonicalize_key_material([2, 1]) + + def test_drops_function_valued_object_entries_matching_json_semantics(self) -> None: + # Upstream drops `undefined` (and function/symbol) entries. Python has no + # `undefined`; callables are the remaining JS-only analog and are dropped + # from objects / serialized as null in arrays, as JSON.stringify does. + assert canonicalize_key_material({"a": 1, "b": len}) == canonicalize_key_material({"a": 1}) + assert canonicalize_key_material([1, len]) == "[1,null]" + + def test_canonicalizes_primitives_per_jcs(self) -> None: + assert canonicalize_key_material("query") == '"query"' + assert canonicalize_key_material(42) == "42" + assert canonicalize_key_material(None) == "null" + # JCS number serialization is ECMAScript JSON.stringify: + assert canonicalize_key_material(-0.0) == "0" + assert canonicalize_key_material(1e21) == "1e+21" + + def test_number_formatting_follows_ecmascript(self) -> None: + # Python-specific: ints go through IEEE-754 doubles, exactly as + # JSON.parse would have rounded them on the TypeScript side. + assert canonicalize_key_material(2**53 + 1) == "9007199254740992" + assert canonicalize_key_material(1.0) == "1" + assert canonicalize_key_material(1e-7) == "1e-7" + assert canonicalize_key_material(0.000001) == "0.000001" + assert canonicalize_key_material(1.5e20) == "150000000000000000000" + assert canonicalize_key_material(True) == "true" + assert canonicalize_key_material(False) == "false" + + def test_sorts_keys_by_utf16_code_units(self) -> None: + # U+FF61 sorts AFTER U+1F600 by code point but BEFORE it by UTF-16 code + # unit (0xFF61 > 0xD83D). JCS mandates UTF-16 order. + assert canonicalize_key_material({"\U0001f600": 1, "\uff61": 2}) == '{"\U0001f600":1,"\uff61":2}' + + def test_rejects_values_rfc8785_cannot_represent(self) -> None: + with pytest.raises(ValueError, match=re.compile("non-finite", re.I)): + canonicalize_key_material(float("nan")) + with pytest.raises(ValueError, match=re.compile("non-finite", re.I)): + canonicalize_key_material(float("inf")) + # Upstream's third case is a `bigint`. Python ints are not bigints, but + # an int outside the double range is equally unrepresentable. + with pytest.raises(ValueError, match=re.compile("non-finite", re.I)): + canonicalize_key_material({"id": 10**400}) + + def test_rejects_non_json_types(self) -> None: + with pytest.raises(TypeError): + canonicalize_key_material({"blob": b"bytes"}) + with pytest.raises(TypeError): + canonicalize_key_material({1: "non-string key"}) + + def test_throws_on_circular_key_material_instead_of_hanging(self) -> None: + circular: Dict[str, Any] = {} + circular["self"] = circular + with pytest.raises(ValueError, match=re.compile("circular", re.I)): + canonicalize_key_material(circular) + + def test_shared_non_circular_references_are_fine(self) -> None: + shared = {"x": 1} + assert canonicalize_key_material({"a": shared, "b": shared}) == '{"a":{"x":1},"b":{"x":1}}' + + def test_throws_on_nesting_past_max_depth_instead_of_overflowing(self) -> None: + deep: Any = "leaf" + for _ in range(MAX_CANONICALIZE_DEPTH + 10): + deep = [deep] + with pytest.raises(ValueError, match=re.compile("deeper than", re.I)): + canonicalize_key_material(deep) + + +# --------------------------------------------------------------------------- +# Fingerprints — the cross-port contract +# --------------------------------------------------------------------------- + + +class TestFingerprints: + async def test_reproduces_every_tool_call_vector(self) -> None: + assert len(VECTORS["toolCallVectors"]) == 10 + for vector in VECTORS["toolCallVectors"]: + assert canonicalize_key_material(vector["keyMaterial"]) == vector["jcs"], vector["name"] + assert await fingerprint_tool_call(vector["toolName"], vector["keyMaterial"]) == vector["fingerprint"], ( + vector["name"] + ) + + async def test_reproduces_every_bare_key_material_vector(self) -> None: + assert len(VECTORS["keyMaterialVectors"]) == 2 + for vector in VECTORS["keyMaterialVectors"]: + assert canonicalize_key_material(vector["keyMaterial"]) == vector["jcs"], vector["name"] + assert await fingerprint_key_material(vector["keyMaterial"]) == vector["fingerprint"], vector["name"] + + async def test_rejected_vector_classes_raise(self) -> None: + names = [entry["name"] for entry in VECTORS["rejected"]] + assert names == ["bigint", "NaN / Infinity", "circular reference", "nesting > 64 levels"] + circular: List[Any] = [] + circular.append(circular) + deep: Any = 0 + for _ in range(MAX_CANONICALIZE_DEPTH + 1): + deep = [deep] + # "bigint" has no Python type; an int outside the double range stands in. + for rejected in (10**400, float("nan"), circular, deep): + with pytest.raises(ValueError): + await fingerprint_tool_call("t", rejected) + + async def test_produces_64_char_lowercase_hex(self) -> None: + fp = await fingerprint_tool_call("t", {}) + assert re.fullmatch(r"[0-9a-f]{64}", fp) + + async def test_includes_the_tool_name(self) -> None: + assert await fingerprint_tool_call("web_search", {"q": "x"}) != await fingerprint_tool_call( + "fetch_page", {"q": "x"} + ) + + async def test_non_ascii_key_material_hashes_over_utf8_bytes(self) -> None: + matching = [v for v in VECTORS["toolCallVectors"] if "non-ascii" in v["name"]] + assert len(matching) == 1 + vector = matching[0] + assert await fingerprint_tool_call(vector["toolName"], vector["keyMaterial"]) == vector["fingerprint"] + + async def test_lone_surrogate_vector_is_escaped(self) -> None: + matching = [v for v in VECTORS["toolCallVectors"] if "lone surrogate" in v["name"]] + assert len(matching) == 1 + assert matching[0]["keyMaterial"] == {"s": "\ud800"} + assert canonicalize_key_material(matching[0]["keyMaterial"]) == '{"s":"\\ud800"}' + + async def test_surrogate_pair_code_points_combine_like_utf16(self) -> None: + # A pair stored as two Python code points is ONE valid character in JS. + assert canonicalize_key_material("\ud83c\udf89") == canonicalize_key_material("\U0001f389") + + +# --------------------------------------------------------------------------- +# loop_key resolution (function | False | absent | field list) +# --------------------------------------------------------------------------- + +ARGS: Dict[str, Any] = {"command": "ls", "cwd": "/tmp", "verbose": True} + + +class TestResolveLoopKeyMaterial: + def test_absent_declaration_is_full_arguments(self) -> None: + assert resolve_loop_key_material(None, ARGS) == {"kind": "key", "key_material": ARGS} + + def test_false_is_statically_exempt(self) -> None: + assert resolve_loop_key_material(False, ARGS) == {"kind": "exempt"} + + def test_field_array_is_declarative_subset(self) -> None: + assert resolve_loop_key_material(["command", "cwd", "not_a_field"], ARGS) == { + "kind": "key", + "key_material": {"command": "ls", "cwd": "/tmp"}, + } + + def test_empty_field_array_warns_and_falls_back(self) -> None: + resolution = resolve_loop_key_material([], ARGS) + assert resolution["kind"] == "fallback" + assert resolution["key_material"] is ARGS + assert "empty field list" in resolution["warning"] + + def test_empty_field_array_does_not_collapse_unrelated_calls(self) -> None: + ls = resolve_loop_key_material([], {"command": "ls"}) + rm = resolve_loop_key_material([], {"command": "rm -rf /"}) + assert ls["kind"] == "fallback" and rm["kind"] == "fallback" + assert ls["key_material"] != rm["key_material"] + + def test_field_array_whose_every_field_is_absent_warns_and_falls_back(self) -> None: + resolution = resolve_loop_key_material(["nope", "also_nope"], ARGS) + assert resolution["kind"] == "fallback" + assert resolution["key_material"] is ARGS + assert "absent from the arguments" in resolution["warning"] + assert "[nope, also_nope]" in resolution["warning"] + + def test_preserves_proto_as_a_declared_field(self) -> None: + # Upstream guards against JS prototype pollution; for Python it is just + # an ordinary key and must survive the subset. + args = json.loads('{"__proto__":"declared"}') + assert resolve_loop_key_material(["__proto__"], args) == { + "kind": "key", + "key_material": {"__proto__": "declared"}, + } + + def test_function_returning_a_value_is_that_value(self) -> None: + resolution = resolve_loop_key_material(lambda a: str(a["command"]).strip(), ARGS) + assert resolution == {"kind": "key", "key_material": "ls"} + + def test_function_returning_none_is_per_call_exemption(self) -> None: + # Upstream: `null` exempts, `undefined` falls back with a warning. + # Python has only `None`, which maps to upstream `null` (exempt) — + # deliberate divergence documented in doom_loop.py's module docstring. + # Upstream's "function returning undefined → FALLBACK" case has no + # Python counterpart. + assert resolve_loop_key_material(lambda a: None, ARGS) == {"kind": "exempt"} + + def test_throwing_function_falls_back_with_warning(self) -> None: + def broken(_: Dict[str, Any]) -> Any: + raise RuntimeError("loopKey bug") + + resolution = resolve_loop_key_material(broken, ARGS) + assert resolution["kind"] == "fallback" + assert resolution["key_material"] is ARGS + assert re.search("threw", resolution["warning"]) + assert "loopKey bug" in resolution["warning"] + + def test_unsupported_shape_falls_back(self) -> None: + # A bare string is not a field list (upstream: typeof 'string' → unsupported). + for shape in ("command", True, 3): + resolution = resolve_loop_key_material(shape, ARGS) + assert resolution["kind"] == "fallback" + assert "unsupported shape" in resolution["warning"] + + +# --------------------------------------------------------------------------- +# Text repetition +# --------------------------------------------------------------------------- + + +class TestDetectTextRepetition: + def test_detects_a_single_repeated_token(self) -> None: + result = detect_text_repetition("no no no no no no no no no no no no") + assert result == {"repeats": 12, "period_tokens": 1, "covered_tokens": 12, "sample": "no"} + + def test_detects_a_repeating_phrase_block(self) -> None: + phrase = "I am stuck in a loop." + result = detect_text_repetition(" ".join([phrase] * 6)) + assert result is not None + assert result["period_tokens"] == 6 + assert result["repeats"] == 6 + assert result["sample"] == phrase + + def test_only_counts_repetition_at_the_tail(self) -> None: + looping = " ".join(["retry now"] * 6) + recovered = f"{looping} but then I found the actual answer to your question about databases" + assert detect_text_repetition(recovered) is None + + def test_stays_quiet_on_ordinary_prose(self) -> None: + assert ( + detect_text_repetition("The quick brown fox jumps over the lazy dog and then runs far away into the woods.") + is None + ) + + def test_stays_quiet_below_the_repeat_threshold(self) -> None: + assert detect_text_repetition("this is very very very good and quite long enough overall") is None + + def test_documented_miss_paraphrased_repetition(self) -> None: + paraphrased = " ".join( + [ + "I am unable to find the file.", + "I cannot locate the file.", + "I am not able to find that file.", + "The file cannot be found by me.", + "Finding the file is not possible.", + "I could not locate that file.", + ] + ) + assert detect_text_repetition(paraphrased) is None + + def test_is_deterministic(self) -> None: + text = " ".join(["loop detected"] * 8) + first = detect_text_repetition(text) + assert first is not None + assert first == detect_text_repetition(text) + + def test_handles_multi_megabyte_input_via_tail_slice(self) -> None: + filler = " ".join(f"w{i}" for i in range(400_000)) + doom_tail = " ".join(["stuck"] * 20) + result = detect_text_repetition(f"{filler} {doom_tail}") + _subset(result, {"period_tokens": 1, "repeats": 20}) + + def test_honors_custom_thresholds(self) -> None: + text = "ha ha ha" + assert detect_text_repetition(text) is None + _subset( + detect_text_repetition(text, {"min_repeats": 3, "min_covered_tokens": 3}), + {"repeats": 3, "period_tokens": 1}, + ) + + def test_prefers_smallest_period_on_ties(self) -> None: + result = detect_text_repetition("no no no no no no", {"min_repeats": 3, "min_covered_tokens": 6}) + _subset(result, {"period_tokens": 1, "repeats": 6}) + + +# --------------------------------------------------------------------------- +# Ladder +# --------------------------------------------------------------------------- + +LADDER: DoomLoopLadder = {"observe": 2, "steer": False, "block": 3, "stop": 6} + + +class TestResolveLadderAction: + def test_maps_streaks_onto_rungs_strongest_wins(self) -> None: + assert [resolve_ladder_action(LADDER, streak, allow_block=True) for streak in (1, 2, 3, 5, 6)] == [ + None, + "observe", + "block", + "block", + "stop", + ] + + def test_falls_through_block_for_non_blockable_verdicts(self) -> None: + assert resolve_ladder_action(LADDER, 3, allow_block=False) == "observe" + assert resolve_ladder_action(LADDER, 6, allow_block=False) == "stop" + + def test_respects_disabled_rungs(self) -> None: + assert ( + resolve_ladder_action({"observe": False, "steer": False, "block": False, "stop": 4}, 3, allow_block=True) + is None + ) + + def test_escalate_rung_and_its_gate(self) -> None: + ladder: DoomLoopLadder = {"observe": 1, "steer": 2, "escalate": 3, "block": 5, "stop": 8} + assert resolve_ladder_action(ladder, 3, allow_block=True) == "escalate" + assert resolve_ladder_action(ladder, 3, allow_block=True, allow_escalate=False) == "steer" + assert resolve_ladder_action(ladder, 5, allow_block=False) == "escalate" + + +class TestResolveDoomLoopOption: + def test_returns_none_for_none_or_false(self) -> None: + assert resolve_doom_loop_option(None) is None + assert resolve_doom_loop_option(False) is None + + def test_true_resolves_to_the_documented_defaults(self) -> None: + resolved = resolve_doom_loop_option(True) + assert resolved["ladder"] == DEFAULT_DOOM_LOOP_LADDER + assert resolved["ladder"] == {"observe": 2, "steer": False, "escalate": False, "block": 3, "stop": 6} + assert resolved["text"] == { + "enabled": True, + "max_period_tokens": 16, + "min_repeats": 4, + "min_covered_tokens": 12, + "max_window_tokens": 400, + } + assert resolved["escalation"] is None + + def test_default_ladder_is_read_only(self) -> None: + with pytest.raises(TypeError): + DEFAULT_DOOM_LOOP_LADDER["block"] = 1 # type: ignore[index] + + def test_merges_partial_ladders_and_clamps_invalid_thresholds(self) -> None: + resolved = resolve_doom_loop_option({"ladder": {"block": 5, "stop": 0}}) + assert resolved["ladder"]["block"] == 5 + assert resolved["ladder"]["stop"] == DEFAULT_DOOM_LOOP_LADDER["stop"] + assert resolved["ladder"]["observe"] == DEFAULT_DOOM_LOOP_LADDER["observe"] + + def test_floors_fractional_thresholds_and_rejects_bools(self) -> None: + # `True` is an int in Python but `typeof true !== 'number'` upstream. + # Deliberately off-type input (callers may pass untyped config). + option: Any = {"ladder": {"block": 4.7, "observe": True}} + resolved = resolve_doom_loop_option(option) + assert resolved["ladder"]["block"] == 4 + assert resolved["ladder"]["observe"] == 2 + + def test_advisor_false_is_not_a_recovery_mechanism(self) -> None: + assert resolve_doom_loop_option({"escalation": {"advisor": False}})["escalation"] is None + + def test_advisor_false_does_not_resolve_alongside_a_real_mechanism(self) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + resolved = resolve_doom_loop_option({"escalation": {"model": "stronger-model", "advisor": False}}) + escalation = resolved["escalation"] + assert escalation is not None + assert escalation["model"] == "stronger-model" + assert "advisor" not in escalation + + def test_advisor_true_is_a_recovery_mechanism(self) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + escalation = resolve_doom_loop_option({"escalation": {"advisor": True}})["escalation"] + assert escalation == {"advisor": True, "max_escalations": 2} + + def test_escalation_budget_resolution(self) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fractional: Any = {"escalation": {"model": "m", "max_escalations": 3.9}} + assert resolve_doom_loop_option(fractional)["escalation"] == { + "model": "m", + "max_escalations": 3, + } + assert resolve_doom_loop_option({"escalation": {"model": "m", "max_escalations": 0}})["escalation"] == { + "model": "m", + "max_escalations": 2, + } + + def test_warns_when_escalation_config_has_no_rung(self) -> None: + with pytest.warns(UserWarning, match="escalate ladder rung is disabled"): + resolve_doom_loop_option({"escalation": {"model": "m"}}) + + def test_warns_when_escalate_rung_has_no_config(self) -> None: + with pytest.warns(UserWarning, match="no escalation config"): + resolved = resolve_doom_loop_option({"ladder": {"observe": 1, "escalate": 2, "block": 3}}) + assert resolved["escalation"] is None + + def test_text_false_disables_text_detection(self) -> None: + assert resolve_doom_loop_option({"text": False})["text"]["enabled"] is False + + def test_text_options_merge_with_defaults(self) -> None: + text = resolve_doom_loop_option({"text": {"min_repeats": 3}})["text"] + assert text["min_repeats"] == 3 + assert text["max_period_tokens"] == 16 + + def test_warns_when_block_enabled_with_stop_disabled(self) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + resolve_doom_loop_option({"ladder": {"block": 3, "stop": False}}) + messages = [str(w.message) for w in caught] + assert len(messages) == 1 + assert "stop disabled" in messages[0] + + def test_warns_on_dead_rungs(self) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + resolve_doom_loop_option({"ladder": {"observe": 5, "block": 2}}) + messages = [str(w.message) for w in caught] + # One warning per dead rung: observe (5) is dominated by block (2). + assert len(messages) == 1 + assert '"observe" (5) can never fire' in messages[0] + + def test_does_not_warn_on_the_default_ladder(self) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + resolve_doom_loop_option(True) + assert caught == [] + + +# --------------------------------------------------------------------------- +# Monitor state machine (round-scoped streaks) +# --------------------------------------------------------------------------- + + +class TestMonitorToolCallStreaks: + async def test_fires_observe_at_2_and_block_at_3(self) -> None: + monitor = _make_monitor() + args = {"query": "same thing"} + assert "verdict" not in await monitor.record_tool_call("search", args, 1) + second = await monitor.record_tool_call("search", args, 2) + _subset( + second.get("verdict"), + {"action": "observe", "streak": 2, "detector": "tool-fingerprint", "tool_name": "search"}, + ) + third = await monitor.record_tool_call("search", args, 3) + _subset(third.get("verdict"), {"action": "block", "streak": 3}) + verdict = third.get("verdict") + assert verdict is not None + assert verdict["fingerprint"] == await fingerprint_tool_call("search", args) + # Undeclared round: the true round membership is unknown, so the text + # is the fingerprint-free per-call form (upstream buildToolVerdictMessage). + assert verdict["message"] == ( + 'Doom loop suspected: this exact "search" call has been repeated across consecutive rounds. ' + "Repeating it will not change the result. Take a different approach, or explain why " + "repetition is required." + ) + + async def test_declared_single_call_round_quotes_the_fingerprint(self) -> None: + monitor = _make_monitor() + args = {"query": "same thing"} + record: Optional[DoomLoopCallRecord] = None + for round_number in (1, 2, 3): + await monitor.declare_round(round_number, [{"tool_name": "search", "key_material": args}]) + record = await monitor.record_tool_call("search", args, round_number) + assert record is not None + verdict = record.get("verdict") + assert verdict is not None + assert verdict["action"] == "block" + assert verdict["message"] == ( + 'Doom loop suspected: tool "search" was invoked in 3 consecutive rounds with identical ' + "arguments (fingerprint " + verdict["fingerprint"][:16] + "…). Repeating the call will " + "not change the result. Take a different approach, or explain why repetition is required." + ) + + async def test_escalates_to_stop_at_the_stop_threshold(self) -> None: + monitor = _make_monitor() + actions: List[Optional[str]] = [] + last: Optional[DoomLoopCallRecord] = None + for round_number in range(1, 7): + last = await monitor.record_tool_call("search", {"q": "x"}, round_number) + verdict = last.get("verdict") + actions.append(verdict["action"] if verdict else None) + assert last is not None + _subset(last.get("verdict"), {"action": "stop", "streak": 6}) + assert actions == [None, "observe", "block", "block", "block", "stop"] + + async def test_identical_calls_in_the_same_round_are_duplicates(self) -> None: + monitor = _make_monitor() + args = {"q": "parallel"} + first = await monitor.record_tool_call("search", args, 1) + _subset(first, {"streak": 1, "duplicate_in_round": False}) + for _ in range(5): + dup = await monitor.record_tool_call("search", args, 1) + _subset(dup, {"streak": 1, "duplicate_in_round": True}) + assert "verdict" not in dup + _subset(await monitor.record_tool_call("search", args, 2), {"streak": 2, "duplicate_in_round": False}) + + async def test_different_arguments_reset_the_streak(self) -> None: + monitor = _make_monitor() + await monitor.record_tool_call("search", {"q": "a"}, 1) + await monitor.record_tool_call("search", {"q": "a"}, 2) + reset = await monitor.record_tool_call("search", {"q": "b"}, 3) + assert "verdict" not in reset + assert reset["streak"] == 1 + _subset((await monitor.record_tool_call("search", {"q": "b"}, 4)).get("verdict"), {"streak": 2}) + + async def test_interleaved_calls_to_other_tools_do_not_reset(self) -> None: + monitor = _make_monitor() + search = {"q": "x"} + await monitor.record_tool_call("search", search, 1) + await monitor.record_tool_call("read_file", {"path": "/a"}, 2) + _subset( + (await monitor.record_tool_call("search", search, 3)).get("verdict"), + {"action": "observe", "streak": 2}, + ) + await monitor.record_tool_call("read_file", {"path": "/b"}, 4) + _subset( + (await monitor.record_tool_call("search", search, 5)).get("verdict"), + {"action": "block", "streak": 3}, + ) + + async def test_argument_key_order_does_not_defeat_detection(self) -> None: + monitor = _make_monitor() + await monitor.record_tool_call("run", {"cmd": "ls", "cwd": "/tmp"}, 1) + _subset((await monitor.record_tool_call("run", {"cwd": "/tmp", "cmd": "ls"}, 2)).get("verdict"), {"streak": 2}) + + async def test_repeated_empty_calls_trip_the_detector(self) -> None: + monitor = _make_monitor() + assert "verdict" not in await monitor.record_tool_call("list_tasks", {}, 1) + _subset( + (await monitor.record_tool_call("list_tasks", {}, 2)).get("verdict"), + {"action": "observe", "streak": 2}, + ) + _subset( + (await monitor.record_tool_call("list_tasks", {}, 3)).get("verdict"), + {"action": "block", "streak": 3}, + ) + + async def test_propagates_canonicalization_errors_for_engine_fallback(self) -> None: + # Upstream uses a bigint; an out-of-double-range int is the Python analog. + monitor = _make_monitor() + with pytest.raises(ValueError, match=re.compile("non-finite", re.I)): + await monitor.record_tool_call("search", {"id": 10**400}, 1) + # A failed record leaves no trace: the next valid call starts at 1. + assert (await monitor.record_tool_call("search", {"id": 1}, 1))["streak"] == 1 + + async def test_server_tool_records_use_non_blockable_path_and_label(self) -> None: + monitor = _make_monitor() + last: Optional[DoomLoopCallRecord] = None + for round_number in range(1, 4): + last = await monitor.record_tool_call( + "server:web_search_call", + {"query": "x"}, + round_number, + allow_block=False, + detector="server-tool-fingerprint", + ) + assert last is not None + _subset(last.get("verdict"), {"detector": "server-tool-fingerprint", "action": "observe", "streak": 3}) + + async def test_is_deterministic_across_monitors(self) -> None: + script: List[Tuple[str, Any, int]] = [ + ("a", {"x": 1}, 1), + ("b", {}, 1), + ("a", {"x": 1}, 2), + ("a", {"x": 2}, 3), + ("a", {"x": 2}, 4), + ] + + async def run() -> List[Optional[DoomLoopVerdict]]: + monitor = _make_monitor() + return [(await monitor.record_tool_call(name, args, r)).get("verdict") for name, args, r in script] + + first = await run() + assert first == await run() + assert [v["action"] if v else None for v in first] == [None, None, "observe", None, "observe"] + + +class TestMonitorRoundSets: + """Round-set / per-call semantics (exercised upstream mainly via the fan-out suite).""" + + async def _round( + self, monitor: DoomLoopMonitor, round_number: int, paths: List[str], declare: bool = True + ) -> List[DoomLoopCallRecord]: + if declare: + await monitor.declare_round( + round_number, [{"tool_name": "read", "key_material": {"path": p}} for p in paths] + ) + return [await monitor.record_tool_call("read", {"path": p}, round_number) for p in paths] + + async def test_declared_fan_out_shares_the_round_streak_and_message(self) -> None: + monitor = _make_monitor() + await self._round(monitor, 1, ["a", "b", "c"]) + await self._round(monitor, 2, ["a", "b", "c"]) + third = await self._round(monitor, 3, ["c", "a", "b"]) # emission order does not matter + assert [r["streak"] for r in third] == [3, 3, 3] + verdicts = [r.get("verdict") for r in third] + assert [v["action"] if v else None for v in verdicts] == ["block", "block", "block"] + messages = {v["message"] for v in verdicts if v} + assert len(messages) == 1 + assert "same set of 3 parallel calls" in messages.pop() + + async def test_superset_round_resets_and_does_not_block_progress(self) -> None: + monitor = _make_monitor() + await self._round(monitor, 1, ["a", "b"]) + second = await self._round(monitor, 2, ["a", "b"]) + assert [r["streak"] for r in second] == [2, 2] + third = await self._round(monitor, 3, ["a", "b", "c"]) + # Round set differs (reset to 1) but a and b are 3-peats per call. + assert [r["streak"] for r in third] == [3, 3, 1] + actions = [r["verdict"]["action"] if "verdict" in r else None for r in third] + assert actions == ["block", "block", None] + + async def test_per_call_streak_catches_repeat_in_varying_company(self) -> None: + monitor = _make_monitor() + results = [] + for round_number, other in enumerate(["b", "c", "d"], start=1): + results.append(await self._round(monitor, round_number, ["a", other])) + last_a, last_other = results[-1] + assert last_a["streak"] == 3 + assert last_other["streak"] == 1 + verdict = last_a.get("verdict") + assert verdict is not None + assert verdict["action"] == "block" + # Per-call verdict text is fingerprint- and count-free. + assert verdict["message"] == ( + 'Doom loop suspected: this exact "read" call has been repeated across consecutive rounds. ' + "Repeating it will not change the result. Take a different approach, or explain why " + "repetition is required." + ) + + async def test_undeclared_round_uses_per_call_text(self) -> None: + monitor = _make_monitor() + for round_number in (1, 2): + records = await self._round(monitor, round_number, ["a", "b"], declare=False) + messages = [r["verdict"]["message"] for r in records if "verdict" in r] + assert len(messages) == 2 + assert len(set(messages)) == 1 + assert 'this exact "read" call' in messages[0] + + async def test_unhashable_declared_call_is_excluded_with_warning(self) -> None: + monitor = _make_monitor() + with pytest.warns(UserWarning, match='could not fingerprint a "read" call while declaring round 1'): + await monitor.declare_round( + 1, + [ + {"tool_name": "read", "key_material": {"path": "a"}}, + {"tool_name": "read", "key_material": {"bad": float("nan")}}, + ], + ) + record = await monitor.record_tool_call("read", {"path": "a"}, 1) + assert record["streak"] == 1 + + async def test_non_member_does_not_overwrite_round_identity(self) -> None: + monitor = _make_monitor() + for round_number in (1, 2, 3): + await monitor.declare_round(round_number, [{"tool_name": "read", "key_material": {"path": "a"}}]) + member = await monitor.record_tool_call("read", {"path": "a"}, round_number) + await monitor.record_tool_call("read", {"path": "fallback"}, round_number) + assert member["streak"] == 3 + assert member.get("verdict", {}).get("action") == "block" + state = monitor.get_state() + # The persisted identity pairs with the round streak: the member, not the non-member. + assert state["tools"]["read"]["fingerprint"] == await fingerprint_tool_call("read", {"path": "a"}) + + +class TestMonitorEscalation: + def _config(self) -> Any: + return { + "ladder": {"observe": False, "escalate": 2, "block": False, "stop": 10}, + "escalation": {"model": "frontier", "max_escalations": 1}, + } + + def _monitor(self) -> DoomLoopMonitor: + return _make_monitor(self._config()) + + async def test_escalate_until_budget_is_consumed_then_falls_through(self) -> None: + monitor = self._monitor() + assert monitor.can_escalate() is True + await monitor.record_tool_call("search", {"q": "x"}, 1) + second = await monitor.record_tool_call("search", {"q": "x"}, 2) + assert second.get("verdict", {}).get("action") == "escalate" + monitor.consume_escalation() + assert monitor.can_escalate() is False + third = await monitor.record_tool_call("search", {"q": "x"}, 3) + # Budget exhausted; steer/observe disabled → no verdict. + assert "verdict" not in third + + async def test_escalation_budget_survives_state_round_trip(self) -> None: + monitor = self._monitor() + monitor.consume_escalation() + state = json.loads(json.dumps(monitor.get_state())) + assert state["escalations_used"] == 1 + resumed = DoomLoopMonitor(resolve_doom_loop_option(self._config()), state) + assert resumed.can_escalate() is False + + async def test_no_mechanism_means_never_escalate(self) -> None: + assert _make_monitor().can_escalate() is False + + +class TestMonitorText: + async def test_within_response_repetition_can_stop_immediately(self) -> None: + monitor = _make_monitor() + verdict = await monitor.record_assistant_text(" ".join(["I am stuck."] * 20)) + _subset(verdict, {"detector": "text-repetition", "action": "stop", "streak": 20}) + assert verdict is not None + assert verdict["fingerprint"] == await fingerprint_key_material("I am stuck.") + assert "tool_name" not in verdict + + async def test_cross_step_identical_text_builds_a_streak(self) -> None: + monitor = _make_monitor() + assert await monitor.record_assistant_text("Let me try that again.") is None + _subset( + await monitor.record_assistant_text("Let me try that again."), + {"detector": "text-streak", "action": "observe", "streak": 2}, + ) + # Streak 3 crosses the block rung, but text cannot be blocked → observe. + _subset( + await monitor.record_assistant_text("Let me try that again."), + {"detector": "text-streak", "action": "observe", "streak": 3}, + ) + for _ in range(2): + await monitor.record_assistant_text("Let me try that again.") + _subset(await monitor.record_assistant_text("Let me try that again."), {"action": "stop", "streak": 6}) + + async def test_whitespace_only_differences_do_not_defeat_the_streak(self) -> None: + monitor = _make_monitor() + await monitor.record_assistant_text("Retrying the\nsame plan.") + verdict = await monitor.record_assistant_text("Retrying the same plan.") + _subset(verdict, {"streak": 2}) + assert verdict is not None + # Cross-port: the whitespace-normalized text vector. + assert verdict["fingerprint"] == VECTORS["keyMaterialVectors"][1]["fingerprint"] + + async def test_documented_miss_paraphrased_cross_step_text(self) -> None: + monitor = _make_monitor() + paraphrases = [ + "I am unable to find the file.", + "I cannot locate the file.", + "I am not able to find that file.", + "The file cannot be found.", + "Locating the file is not possible.", + "I could not find that file.", + ] + assert [await monitor.record_assistant_text(text) for text in paraphrases] == [None] * 6 + + async def test_empty_text_neither_counts_nor_resets(self) -> None: + monitor = _make_monitor() + await monitor.record_assistant_text("same") + assert await monitor.record_assistant_text("") is None + assert await monitor.record_assistant_text(" ") is None + _subset(await monitor.record_assistant_text("same"), {"streak": 2}) + + async def test_reset_text_streak(self) -> None: + monitor = _make_monitor() + await monitor.record_assistant_text("same") + monitor.reset_text_streak() + assert await monitor.record_assistant_text("same") is None + assert "text" in monitor.get_state() + + async def test_text_false_disables_both_text_detectors(self) -> None: + monitor = _make_monitor({"text": False}) + assert await monitor.record_assistant_text(" ".join(["loop"] * 30)) is None + + +class TestMonitorStateRoundTrip: + async def test_serializes_to_plain_json_and_resumes_counting(self) -> None: + monitor = _make_monitor() + await monitor.record_tool_call("search", {"q": "x"}, 1) + await monitor.record_tool_call("search", {"q": "x"}, 2) + blob = json.loads(json.dumps(monitor.get_state())) + resumed = DoomLoopMonitor(resolve_doom_loop_option(True), blob) + # Round markers are NOT serialized, so the first resumed record increments. + _subset( + (await resumed.record_tool_call("search", {"q": "x"}, 1)).get("verdict"), + {"action": "block", "streak": 3}, + ) + + async def test_ignores_corrupt_persisted_state(self) -> None: + config = resolve_doom_loop_option(True) + with pytest.warns(UserWarning, match="Ignoring invalid persisted doom-loop state"): + monitor = DoomLoopMonitor(config, "not an object") + record = await monitor.record_tool_call("search", {"q": "x"}, 1) + assert record["streak"] == 1 + assert "verdict" not in record + + async def test_persisted_proto_tool_name_cannot_pollute_bookkeeping(self) -> None: + hostile = json.loads('{"tools":{"__proto__":{"fingerprint":"x","streak":999}}}') + monitor = DoomLoopMonitor(resolve_doom_loop_option(True), hostile) + record = await monitor.record_tool_call("unrelated_tool", {"q": "first call"}, 1) + assert record["streak"] == 1 + assert "verdict" not in record + + async def test_a_tool_named_proto_is_tracked_as_ordinary_data(self) -> None: + monitor = _make_monitor() + await monitor.record_tool_call("__proto__", {}, 1) + record = await monitor.record_tool_call("__proto__", {}, 2) + assert record["streak"] == 2 + snapshot = monitor.get_state() + assert list(snapshot["tools"]) == ["__proto__"] + revived = json.loads(json.dumps(snapshot)) + assert list(revived["tools"]) == ["__proto__"] + + async def test_get_state_returns_a_snapshot_not_a_live_reference(self) -> None: + monitor = _make_monitor() + await monitor.record_tool_call("a", {}, 1) + snapshot = monitor.get_state() + await monitor.record_tool_call("a", {}, 2) + assert snapshot["tools"]["a"]["streak"] == 1 + assert monitor.get_state()["tools"]["a"]["streak"] == 2 + + async def test_get_state_shape(self) -> None: + monitor = _make_monitor() + await monitor.declare_round(1, [{"tool_name": "read", "key_material": {"path": p}} for p in ("a", "b")]) + for p in ("a", "b"): + await monitor.record_tool_call("read", {"path": p}, 1) + await monitor.record_tool_call("search", {"q": "x"}, 1) + await monitor.record_assistant_text("hello") + state = monitor.get_state() + fa = await fingerprint_tool_call("read", {"path": "a"}) + fb = await fingerprint_tool_call("read", {"path": "b"}) + assert state == { + "tools": { + "read": {"fingerprint": fb, "streak": 1, "round_fingerprints": sorted([fa, fb])}, + "search": {"fingerprint": await fingerprint_tool_call("search", {"q": "x"}), "streak": 1}, + }, + "text": {"fingerprint": await fingerprint_key_material("hello"), "streak": 1}, + } + + async def test_shrunk_round_persists_call_streaks_across_resume(self) -> None: + monitor = _make_monitor() + for round_number in (1, 2): + await monitor.declare_round( + round_number, [{"tool_name": "read", "key_material": {"path": p}} for p in ("a", "b")] + ) + for p in ("a", "b"): + await monitor.record_tool_call("read", {"path": p}, round_number) + # Round shrinks to one call: round streak resets, per-call count keeps climbing. + await monitor.declare_round(3, [{"tool_name": "read", "key_material": {"path": "a"}}]) + await monitor.record_tool_call("read", {"path": "a"}, 3) + state = json.loads(json.dumps(monitor.get_state())) + fa = await fingerprint_tool_call("read", {"path": "a"}) + assert state["tools"]["read"] == {"fingerprint": fa, "streak": 1, "call_streaks": {fa: 3}} + resumed = DoomLoopMonitor(resolve_doom_loop_option(True), state) + record = await resumed.record_tool_call("read", {"path": "a"}, 1) + assert record["streak"] == 4 + assert record.get("verdict", {}).get("action") == "block" + + async def test_restore_accepts_upstream_camel_case_blob(self) -> None: + fa = await fingerprint_tool_call("read", {"path": "a"}) + fb = await fingerprint_tool_call("read", {"path": "b"}) + blob = { + "tools": {"read": {"fingerprint": fb, "streak": 2, "roundFingerprints": [fb, fa]}}, + "escalationsUsed": 1, + } + monitor = DoomLoopMonitor(resolve_doom_loop_option(True), blob) + await monitor.declare_round(1, [{"tool_name": "read", "key_material": {"path": p}} for p in ("a", "b")]) + streaks = [(await monitor.record_tool_call("read", {"path": p}, 1))["streak"] for p in ("a", "b")] + assert streaks == [3, 3] + assert monitor.get_state().get("escalations_used") == 1 + + async def test_restore_clamps_corrupt_streaks_and_drops_invalid_entries(self) -> None: + blob = { + "tools": { + "neg": {"fingerprint": "x", "streak": -50}, + "bad": {"fingerprint": 1, "streak": 2}, + "boolean": {"fingerprint": "x", "streak": True}, + }, + "text": {"fingerprint": "t", "streak": "nope"}, + "escalations_used": -1, + } + monitor = DoomLoopMonitor(resolve_doom_loop_option(True), blob) + assert monitor.get_state() == {"tools": {"neg": {"fingerprint": "x", "streak": 1}}} diff --git a/tests/unit/test_doom_loop_escalation.py b/tests/unit/test_doom_loop_escalation.py new file mode 100644 index 0000000..b7fb23d --- /dev/null +++ b/tests/unit/test_doom_loop_escalation.py @@ -0,0 +1,513 @@ +"""Tests for the doom-loop `escalate` recovery rung. + +Ports `packages/agent/tests/unit/doom-loop-escalation.test.ts`. + +On detection, throw more intelligence at the NEXT turn -- a one-turn model swap +and/or a forced `openrouter:advisor` consult -- then return to the configured +model. All scenarios simulate the LLM with scripted `QueuedClient` responses. + +Port notes: +- Request keys are the generated Python SDK's snake_case: `tool_choice` + (upstream `toolChoice`) and the advisor's `forward_transcript` (upstream + `forwardTranscript`; see `AdvisorServerToolConfig`). Deliberate. +- Config/state keys are snake_case (`max_escalations`, `escalations_used`). +- Upstream spies on `console.warn`; this port warns via `warnings.warn`. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Mapping + +import pytest + +from openrouter_agent import ( + DoomLoopLadder, + DoomLoopMonitor, + HookEntry, + HooksManager, + call_model, + resolve_doom_loop_option, + tool, +) +from tests._fixtures import MemoryStateAccessor, QueuedClient, function_call_item, make_response, message_item + +pytestmark = pytest.mark.filterwarnings("ignore:\\[DoomLoop\\]:UserWarning") + + +class Script: + def __init__(self) -> None: + self.counter = 0 + + def response(self, output: List[Dict[str, Any]]) -> Dict[str, Any]: + self.counter += 1 + return make_response(f"resp_{self.counter}", output) + + def call(self, name: str, args: Any) -> Dict[str, Any]: + self.counter += 1 + return function_call_item(f"call_{self.counter}", name, json.dumps(args)) + + def message(self, text: str) -> Dict[str, Any]: + self.counter += 1 + return message_item(text, item_id=f"msg_{self.counter}") + + def tool_call_turn(self, name: str, args: Any) -> Dict[str, Any]: + return self.response([self.call(name, args)]) + + def text_turn(self, text: str) -> Dict[str, Any]: + return self.response([self.message(text)]) + + def text_and_call_turn(self, text: str, query: str) -> Dict[str, Any]: + return self.response([self.message(text), self.call("web_search", {"query": query})]) + + +def collect_detections(hooks: HooksManager) -> List[Dict[str, Any]]: + detections: List[Dict[str, Any]] = [] + + def handler(payload: Dict[str, Any], ctx: Any) -> None: + detections.append(payload) + + hooks.on("DoomLoopDetected", HookEntry(handler=handler)) + return detections + + +search_tool = tool( + name="web_search", + input_schema=dict, + output_schema=dict, + execute=lambda p, c: {"results": f"nothing for {p['query']}"}, +) + +#: Escalate at streak 2, block/stop far away -- isolates the recovery rung. +ESCALATE_AT_2: DoomLoopLadder = {"observe": 1, "steer": False, "escalate": 2, "block": 5, "stop": 8} + + +def advisor_of(request: Mapping[str, Any]) -> Dict[str, Any]: + tools = request.get("tools") or [] + matches = [t for t in tools if t.get("type") == "openrouter:advisor"] + assert len(matches) == 1 + return matches[0] + + +# --------------------------------------------------------------------------- +# Config resolution +# --------------------------------------------------------------------------- + + +def test_resolves_a_mechanism_with_the_default_budget() -> None: + resolved = resolve_doom_loop_option({"ladder": {"escalate": 3}, "escalation": {"model": "openai/gpt-5"}}) + assert resolved["escalation"] == {"model": "openai/gpt-5", "max_escalations": 2} + assert resolved["ladder"]["escalate"] == 3 + + +def test_warns_and_disables_the_rung_when_escalate_is_enabled_without_a_mechanism() -> None: + with pytest.warns(UserWarning, match="no escalation config"): + resolved = resolve_doom_loop_option({"ladder": {"escalate": 3}}) + assert resolved["escalation"] is None + + +def test_warns_when_a_mechanism_is_configured_but_the_rung_is_disabled() -> None: + with pytest.warns(UserWarning, match="escalate ladder rung is disabled"): + resolved = resolve_doom_loop_option({"escalation": {"model": "openai/gpt-5"}}) + # Mechanism kept (hook overrides may still use it); rung stays off. + assert resolved["escalation"] is not None + assert resolved["ladder"]["escalate"] is False + + +def test_clamps_an_invalid_max_escalations_to_the_default() -> None: + resolved = resolve_doom_loop_option( + {"ladder": {"escalate": 2}, "escalation": {"model": "openai/gpt-5", "max_escalations": 0}} + ) + assert resolved["escalation"] is not None + assert resolved["escalation"]["max_escalations"] == 2 + + +# --------------------------------------------------------------------------- +# Monitor: ladder placement and budget +# --------------------------------------------------------------------------- + + +def make_monitor(max_escalations: int = 2) -> DoomLoopMonitor: + config = resolve_doom_loop_option( + {"ladder": ESCALATE_AT_2, "escalation": {"model": "openai/gpt-5", "max_escalations": max_escalations}} + ) + return DoomLoopMonitor(config) + + +def _verdict(record: Mapping[str, Any]) -> Dict[str, Any]: + verdict = record.get("verdict") + assert verdict is not None + return dict(verdict) + + +async def test_fires_escalate_between_steer_and_block_and_repeats_while_budget_remains() -> None: + monitor = make_monitor() + args = {"q": "same"} + first = _verdict(await monitor.record_tool_call("search", args, 1)) + assert (first["action"], first["streak"]) == ("observe", 1) + second = _verdict(await monitor.record_tool_call("search", args, 2)) + assert (second["action"], second["streak"]) == ("escalate", 2) + # Budget not yet consumed (engine consumes at application time). + third = _verdict(await monitor.record_tool_call("search", args, 3)) + assert (third["action"], third["streak"]) == ("escalate", 3) + + +async def test_falls_through_to_weaker_rungs_once_the_budget_is_exhausted() -> None: + monitor = make_monitor(1) + args = {"q": "same"} + await monitor.record_tool_call("search", args, 1) + assert _verdict(await monitor.record_tool_call("search", args, 2))["action"] == "escalate" + monitor.consume_escalation() # engine applied it + assert monitor.can_escalate() is False + # Streak 3: escalate unavailable -> observe (steer disabled, block at 5). + third = _verdict(await monitor.record_tool_call("search", args, 3)) + assert (third["action"], third["streak"]) == ("observe", 3) + await monitor.record_tool_call("search", args, 4) + fifth = _verdict(await monitor.record_tool_call("search", args, 5)) + assert (fifth["action"], fifth["streak"]) == ("block", 5) + + +async def test_escalations_used_round_trips_through_serialized_state() -> None: + monitor = make_monitor(2) + monitor.consume_escalation() + blob = json.loads(json.dumps(monitor.get_state())) + assert blob["escalations_used"] == 1 + + resumed = DoomLoopMonitor( + resolve_doom_loop_option( + {"ladder": ESCALATE_AT_2, "escalation": {"model": "openai/gpt-5", "max_escalations": 2}} + ), + blob, + ) + assert resumed.can_escalate() is True + resumed.consume_escalation() + assert resumed.can_escalate() is False + + +# --------------------------------------------------------------------------- +# Engine: one-turn overrides +# --------------------------------------------------------------------------- + + +async def test_swaps_the_model_for_exactly_one_turn_then_reverts() -> None: + script = Script() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "different now"}), + script.text_turn("recovered"), + ] + ) + + text = await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": {"ladder": ESCALATE_AT_2, "escalation": {"model": "frontier-model"}}, + }, + ).get_text() + + assert text == "recovered" + assert [request["model"] for request in client.requests] == [ + "cheap-model", + "cheap-model", + "frontier-model", # follow-up after the escalate verdict + "cheap-model", # automatic revert + ] + + +async def test_injects_the_advisor_tool_with_pinned_tool_choice_for_the_escalated_turn_only() -> None: + script = Script() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "unblocked"}), + script.text_turn("done"), + ] + ) + + await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": {"ladder": ESCALATE_AT_2, "escalation": {"advisor": True}}, + }, + ).get_text() + + escalated = client.requests[2] + advisor = advisor_of(escalated) + parameters = advisor["parameters"] + assert parameters["forward_transcript"] is True + assert "escalation advisor" in str(parameters["instructions"]) + # The configured function tool is still there, before the advisor. + assert [t.get("name", t["type"]) for t in escalated["tools"]] == ["web_search", "openrouter:advisor"] + assert escalated["tool_choice"] == { + "type": "allowed_tools", + "mode": "required", + "tools": [{"type": "openrouter:advisor"}], + } + # Model unchanged (advisor-only escalation). + assert escalated["model"] == "cheap-model" + # Next dispatch reverts: no advisor, original tool choice. + reverted = client.requests[3] + assert all(t.get("type") != "openrouter:advisor" for t in reverted.get("tools") or []) + assert reverted.get("tool_choice") is None + # The dispatches before the escalation carried no advisor either. + for request in client.requests[:2]: + assert all(t.get("type") != "openrouter:advisor" for t in request.get("tools") or []) + + +async def test_does_not_consume_a_caller_choice_replaced_by_an_advisor_override() -> None: + script = Script() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "unblocked"}), + script.text_turn("done"), + ] + ) + + def tool_choice(ctx: Mapping[str, Any]) -> Any: + return {"type": "function", "name": "web_search"} if ctx["number_of_turns"] >= 2 else "auto" + + await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "tool_choice": tool_choice, + "doom_loop": {"ladder": ESCALATE_AT_2, "escalation": {"advisor": True}}, + }, + ).get_text() + + assert client.requests[2]["tool_choice"]["type"] == "allowed_tools" + assert client.requests[2]["tool_choice"]["mode"] == "required" + # The caller's web_search choice did not reach dispatch 2, so it remains + # armed and is still forced after the one-turn advisor override. + assert client.requests[3]["tool_choice"] == {"type": "function", "name": "web_search"} + + +async def test_advisor_object_config_merges_over_the_defaults() -> None: + script = Script() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.text_turn("done"), + ] + ) + + await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": { + "ladder": ESCALATE_AT_2, + "escalation": {"advisor": {"model": "anthropic/claude-opus-4.6", "forward_transcript": False}}, + }, + }, + ).get_text() + + parameters = advisor_of(client.requests[2])["parameters"] + assert parameters["model"] == "anthropic/claude-opus-4.6" + # Explicit config overrides the forward_transcript default. + assert parameters["forward_transcript"] is False + # Default instructions kept. + assert "escalation advisor" in str(parameters["instructions"]) + + +async def test_a_steer_notice_accompanies_the_escalated_turn_so_the_model_knows_why() -> None: + script = Script() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.text_turn("done"), + ] + ) + + await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": {"ladder": ESCALATE_AT_2, "escalation": {"model": "frontier-model"}}, + }, + ).get_text() + + escalated_input = client.requests[2]["input"] + notices = [ + item + for item in escalated_input + if isinstance(item, Mapping) + and item.get("role") == "user" + and isinstance(item.get("content"), str) + and "escalated turn follows" in item["content"] + ] + assert len(notices) == 1 + + +async def test_consumes_budget_per_applied_escalation_and_falls_through_when_exhausted() -> None: + script = Script() + hooks = HooksManager() + detections = collect_detections(hooks) + client = QueuedClient( + [*[script.tool_call_turn("web_search", {"query": "same"}) for _ in range(6)], script.text_turn("gave up")] + ) + accessor = MemoryStateAccessor() + + await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": { + "ladder": ESCALATE_AT_2, + "escalation": {"model": "frontier-model", "max_escalations": 1}, + }, + "state": accessor, + "hooks": hooks, + }, + ).get_text() + + # The final "gave up" text also crosses observe@1 via the text-streak + # detector; this asserts the TOOL streak ladder. + assert [[d["action"], d["streak"]] for d in detections if d["detector"] == "tool-fingerprint"] == [ + ["observe", 1], + ["escalate", 2], + ["observe", 3], + ["observe", 4], + ["block", 5], + ["block", 6], + ] + # Exactly ONE dispatch used the frontier model. + assert [request["model"] for request in client.requests].count("frontier-model") == 1 + assert len(client.requests) == 7 + # Consumed budget persisted for resumes. + assert accessor.stored is not None + assert accessor.stored.doom_loop is not None + assert accessor.stored.doom_loop["escalations_used"] == 1 + + +async def test_hook_override_action_escalate_is_honored_with_budget_and_downgraded_without_config() -> None: + ladder = {"observe": 2, "block": 5, "stop": 8} + + script = Script() + hooks = HooksManager() + hooks.on("DoomLoopDetected", HookEntry(handler=lambda p, c: {"override_action": "escalate"})) + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.text_turn("done"), + ] + ) + await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": {"ladder": ladder, "escalation": {"model": "frontier-model"}}, + "hooks": hooks, + }, + ).get_text() + assert client.requests[2]["model"] == "frontier-model" + + # Without a config: the override downgrades to observe -- no crash, no + # model change. + script2 = Script() + hooks2 = HooksManager() + hooks2.on("DoomLoopDetected", HookEntry(handler=lambda p, c: {"override_action": "escalate"})) + client2 = QueuedClient( + [ + script2.tool_call_turn("web_search", {"query": "same"}), + script2.tool_call_turn("web_search", {"query": "same"}), + script2.text_turn("done"), + ] + ) + text = await call_model( + client2, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": {"ladder": ladder}, + "hooks": hooks2, + }, + ).get_text() + assert text == "done" + assert [request["model"] for request in client2.requests] == ["cheap-model", "cheap-model", "cheap-model"] + + +async def test_text_detector_verdicts_can_escalate_too() -> None: + script = Script() + filler = "Let me look somewhere else." + client = QueuedClient( + [ + script.text_and_call_turn(filler, "q1"), + script.text_and_call_turn(filler, "q2"), + script.tool_call_turn("web_search", {"query": "q3"}), + script.text_turn("unblocked"), + ] + ) + + await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": {"ladder": ESCALATE_AT_2, "escalation": {"model": "frontier-model"}, "text": True}, + }, + ).get_text() + + # The text streak hits 2 on the second response; the NEXT follow-up is + # the (only) escalated dispatch. + assert [request["model"] for request in client.requests] == [ + "cheap-model", + "cheap-model", + "frontier-model", + "cheap-model", + ] + + +async def test_two_detectors_firing_in_one_window_escalate_once() -> None: + script = Script() + + def same_everything() -> Dict[str, Any]: + return script.text_and_call_turn("Trying the same thing.", "same") + + accessor = MemoryStateAccessor() + client = QueuedClient([same_everything(), same_everything(), script.text_turn("done")]) + + await call_model( + client, + { + "model": "cheap-model", + "input": "go", + "tools": [search_tool], + "doom_loop": { + "ladder": ESCALATE_AT_2, + "escalation": {"model": "frontier-model", "max_escalations": 5}, + }, + "state": accessor, + }, + ).get_text() + + assert accessor.stored is not None + assert accessor.stored.doom_loop is not None + assert accessor.stored.doom_loop["escalations_used"] == 1 + assert [request["model"] for request in client.requests].count("frontier-model") == 1 diff --git a/tests/unit/test_doom_loop_fanout.py b/tests/unit/test_doom_loop_fanout.py new file mode 100644 index 0000000..df62a62 --- /dev/null +++ b/tests/unit/test_doom_loop_fanout.py @@ -0,0 +1,461 @@ +"""Regression suite for the same-tool fan-out gap. + +Ports `packages/agent/tests/unit/doom-loop-fanout.test.ts` (monitor-level; +no engine involved). + +A streak keyed on a tool's *last* fingerprint cannot see a repeating fan-out: +`read(a), read(b), read(c)` reissued verbatim has a different last call every +round. A round's identity for one tool is therefore the *set* of fingerprints +it was called with; the engine declares that set before scoring any of the +round's calls. Superset, order-permutation, and expanding-fan-out cases guard +the incremental-set order-dependence the first attempt at this fix had. + +Port notes: upstream's unhashable key material is a `bigint` (`1n`); this +port's analog is an integer outside the IEEE-754 double range (`10**400`), +which `canonicalize_key_material` rejects just like a bigint. Persisted keys +are snake_case (`round_fingerprints`, `call_streaks`). +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Sequence + +import pytest + +from openrouter_agent import DoomLoopCallRecord, DoomLoopMonitor, DoomLoopRoundCall, resolve_doom_loop_option + +#: Expected detector diagnostics (ladder hazards, unhashable key material) are +#: asserted where they matter; silence them elsewhere. +pytestmark = pytest.mark.filterwarnings("ignore:\\[DoomLoop\\]:UserWarning") + +#: Unhashable key material (upstream: `{ size: 1n }`). +UNHASHABLE: Dict[str, Any] = {"size": 10**400} + + +def monitor() -> DoomLoopMonitor: + return DoomLoopMonitor(resolve_doom_loop_option(True)) + + +def steer_monitor() -> DoomLoopMonitor: + return DoomLoopMonitor(resolve_doom_loop_option({"ladder": {"steer": 2}})) + + +def read_calls(paths: Sequence[str], tool_name: str = "read") -> List[DoomLoopRoundCall]: + return [{"tool_name": tool_name, "key_material": {"path": path}} for path in paths] + + +async def play_rounds(fanouts: Sequence[Sequence[str]]) -> List[List[str]]: + """Plays each fan-out as one round, declaring the round's calls first -- + the engine does the same at every execution-batch boundary.""" + detector = monitor() + actions: List[List[str]] = [] + for round_index, paths in enumerate(fanouts): + await detector.declare_round(round_index, read_calls(paths)) + round_actions: List[str] = [] + for path in paths: + record = await detector.record_tool_call("read", {"path": path}, round_index) + verdict = record.get("verdict") + round_actions.append(verdict["action"] if verdict is not None else "none") + actions.append(round_actions) + return actions + + +def _action(record: DoomLoopCallRecord) -> str: + verdict = record.get("verdict") + return verdict["action"] if verdict is not None else "none" + + +async def test_accumulates_across_rounds_that_repeat_a_distinct_argument_fan_out() -> None: + actions = await play_rounds([["a", "b", "c"], ["a", "b", "c"], ["a", "b", "c"]]) + # Every call in a repeating round reports that round's streak. + assert actions == [["none", "none", "none"], ["observe", "observe", "observe"], ["block", "block", "block"]] + + +async def test_is_order_insensitive_within_the_round() -> None: + actions = await play_rounds([["a", "b", "c"], ["c", "a", "b"], ["b", "c", "a"]]) + assert actions[1] == ["observe", "observe", "observe"] + assert actions[2] == ["block", "block", "block"] + + +async def test_scores_each_call_individually_when_the_fan_out_membership_changes() -> None: + actions = await play_rounds([["a", "b"], ["a", "b"], ["a", "z"], ["a", "z"]]) + assert actions[1] == ["observe", "observe"] + # The ROUND identity resets, but `a` is on its third consecutive round. + assert actions[2] == ["block", "none"] + assert actions[3] == ["block", "observe"] + + +async def test_flags_the_calls_of_a_partial_repeat_that_actually_repeated() -> None: + actions = await play_rounds([["a", "b", "c"], ["a", "b"]]) + assert actions[1] == ["observe", "observe"] + + +async def test_flags_the_repeated_members_of_a_superset_round_never_the_new_one() -> None: + actions = await play_rounds([["a", "b"], ["a", "b"], ["a", "b", "c"]]) + assert actions[1] == ["observe", "observe"] + assert actions[2] == ["block", "block", "none"] + + +async def test_scores_a_superset_round_the_same_whatever_order_it_is_emitted_in() -> None: + in_order = await play_rounds([["a", "b"], ["a", "b"], ["a", "b", "c"]]) + permuted = await play_rounds([["a", "b"], ["a", "b"], ["c", "a", "b"]]) + assert in_order[2] == ["block", "block", "none"] + assert permuted[2] == ["none", "block", "block"] + + +async def test_scores_an_expanding_fan_out_per_call_repeats_climb_each_new_call_runs_free() -> None: + actions = await play_rounds([["a"], ["a", "b"], ["a", "b", "c"], ["a", "b", "c", "d"]]) + assert actions == [ + ["none"], + ["observe", "none"], + ["block", "observe", "none"], + ["block", "block", "observe", "none"], + ] + + +async def test_leaves_single_call_rounds_behaving_exactly_as_before() -> None: + actions = await play_rounds([["a"], ["a"], ["a"], ["a"]]) + assert [action for round_actions in actions for action in round_actions] == ["none", "observe", "block", "block"] + + +async def test_still_counts_identical_duplicates_within_one_round_only_once() -> None: + detector = monitor() + first = await detector.record_tool_call("read", {"path": "a"}, 0) + second = await detector.record_tool_call("read", {"path": "a"}, 0) + assert first["duplicate_in_round"] is False + assert second["duplicate_in_round"] is True + assert second["streak"] == first["streak"] + + +async def test_keeps_a_resumed_single_call_streak_incrementing_after_restore() -> None: + detector = monitor() + await detector.record_tool_call("read", {"path": "a"}, 0) + await detector.record_tool_call("read", {"path": "a"}, 1) + resumed = DoomLoopMonitor(resolve_doom_loop_option(True), detector.get_state()) + record = await resumed.record_tool_call("read", {"path": "a"}, 0) + assert record["streak"] == 3 + + +async def test_falls_back_to_per_call_scoring_when_a_round_is_never_declared() -> None: + detector = monitor() + await detector.record_tool_call("server:web_search", {"q": "x"}, 0) + repeated = await detector.record_tool_call("server:web_search", {"q": "x"}, 1) + fresh = await detector.record_tool_call("server:web_search", {"q": "y"}, 1) + # The genuine repeat accumulates; the first-ever call does not inherit it. + assert repeated["streak"] == 2 + assert fresh["streak"] == 1 + assert "verdict" not in fresh + + +async def test_scores_an_undeclared_multi_call_round_per_call_order_independently() -> None: + async def undeclared(rounds: Sequence[Sequence[str]]) -> List[List[str]]: + detector = monitor() + out: List[List[str]] = [] + for round_index, paths in enumerate(rounds): + scored: List[str] = [] + for path in paths: + record = await detector.record_tool_call("read", {"path": path}, round_index) + scored.append(f"{record['streak']}:{_action(record)}") + out.append(scored) + return out + + # Both repeated members accumulate, not just the last-recorded one. + assert await undeclared([["a", "b"], ["a", "b"]]) == [["1:none", "1:none"], ["2:observe", "2:observe"]] + # Flipping the order changes nothing about any call's outcome. + assert await undeclared([["a", "b"], ["b", "a"]]) == [["1:none", "1:none"], ["2:observe", "2:observe"]] + # A varying round accumulates on the repeated member wherever it sits. + assert await undeclared([["b", "a"], ["b", "c"], ["b", "d"]]) == [ + ["1:none", "1:none"], + ["2:observe", "1:none"], + ["3:block", "1:none"], + ] + + +async def test_does_not_let_a_call_outside_the_declared_set_inherit_the_round_streak() -> None: + detector = monitor() + hashable = {"path": "a"} + for round_index in (0, 1): + await detector.declare_round( + round_index, + [{"tool_name": "read", "key_material": hashable}, {"tool_name": "read", "key_material": UNHASHABLE}], + ) + await detector.record_tool_call("read", hashable, round_index) + + await detector.declare_round( + 2, [{"tool_name": "read", "key_material": hashable}, {"tool_name": "read", "key_material": UNHASHABLE}] + ) + member = await detector.record_tool_call("read", hashable, 2) + # Recorded with a fallback identity, as the engine would -- for the FIRST time. + dropped = await detector.record_tool_call("read", {"size": "fallback-identity"}, 2) + + assert member["streak"] == 3 + assert dropped["streak"] == 1 + assert "verdict" not in dropped + + +async def test_accumulates_per_call_evidence_for_an_unhashable_call_reissued_verbatim() -> None: + detector = monitor() + dropped_streaks: List[int] = [] + member_streaks: List[int] = [] + for round_index in (0, 1, 2): + await detector.declare_round( + round_index, + [{"tool_name": "read", "key_material": {"path": "a"}}, {"tool_name": "read", "key_material": UNHASHABLE}], + ) + member_streaks.append((await detector.record_tool_call("read", {"path": "a"}, round_index))["streak"]) + dropped_streaks.append( + (await detector.record_tool_call("read", {"size": "fallback-identity"}, round_index))["streak"] + ) + + assert member_streaks == [1, 2, 3] + assert dropped_streaks == [1, 2, 3] + + +async def test_keeps_accumulating_when_an_unhashable_call_rides_along_every_round() -> None: + async def streaks_for(dropped_first: bool) -> List[int]: + detector = monitor() + streaks: List[int] = [] + for round_index in (0, 1, 2, 3): + await detector.declare_round( + round_index, + [ + {"tool_name": "read", "key_material": {"path": "a"}}, + {"tool_name": "read", "key_material": UNHASHABLE}, + ], + ) + if dropped_first: + await detector.record_tool_call("read", {"size": "fallback-identity"}, round_index) + member = await detector.record_tool_call("read", {"path": "a"}, round_index) + if not dropped_first: + await detector.record_tool_call("read", {"size": "fallback-identity"}, round_index) + streaks.append(member["streak"]) + return streaks + + assert await streaks_for(False) == [1, 2, 3, 4] + # Order must not matter: the streak is a function of the sets, not arrival. + assert await streaks_for(True) == [1, 2, 3, 4] + + +async def test_persists_the_streak_against_the_member_that_earned_it_not_a_non_member() -> None: + detector = monitor() + for round_index in (0, 1): + await detector.declare_round( + round_index, + [{"tool_name": "read", "key_material": {"path": "a"}}, {"tool_name": "read", "key_material": UNHASHABLE}], + ) + await detector.record_tool_call("read", {"path": "a"}, round_index) + # Recorded LAST, so it used to become the persisted identity. + await detector.record_tool_call("read", {"size": "fallback-identity"}, round_index) + + # The non-member's OWN per-call count continues (earned evidence). + resumed_non_member = DoomLoopMonitor(resolve_doom_loop_option(True), detector.get_state()) + own_repeat = await resumed_non_member.record_tool_call("read", {"size": "fallback-identity"}, 0) + assert own_repeat["streak"] == 3 + + # A call NEVER recorded before the save inherits nothing on resume. + resumed_fresh = DoomLoopMonitor(resolve_doom_loop_option(True), detector.get_state()) + fresh = await resumed_fresh.record_tool_call("read", {"size": "never-seen-before"}, 0) + assert fresh["streak"] == 1 + assert "verdict" not in fresh + + # And the real repeat keeps the evidence it earned. + resumed_member = DoomLoopMonitor(resolve_doom_loop_option(True), detector.get_state()) + await resumed_member.declare_round(0, read_calls(["a"])) + real_repeat = await resumed_member.record_tool_call("read", {"path": "a"}, 0) + assert real_repeat["streak"] == 3 + + +async def test_keeps_counting_a_recorded_call_when_a_declared_but_never_recorded_member_disappears() -> None: + detector = monitor() + streaks: List[int] = [] + for round_index in (0, 1, 2, 3): + # Rounds 0-1 declare a phantom alongside the real call; 2-3 do not. + paths = ["a", "phantom"] if round_index < 2 else ["a"] + await detector.declare_round(round_index, read_calls(paths)) + streaks.append((await detector.record_tool_call("read", {"path": "a"}, round_index))["streak"]) + assert streaks == [1, 2, 3, 4] + + +async def test_gives_every_call_of_a_repeating_round_the_same_message_so_steer_dedupes() -> None: + detector = steer_monitor() + paths = ["a", "b", "c"] + messages: List[str] = [] + for round_index in (0, 1): + await detector.declare_round(round_index, read_calls(paths)) + for path in paths: + record = await detector.record_tool_call("read", {"path": path}, round_index) + verdict = record.get("verdict") + if round_index == 1 and verdict is not None: + messages.append(verdict["message"]) + + assert len(messages) == 3 + assert len(set(messages)) == 1 + assert "3 parallel calls" in messages[0] + + +async def test_continues_a_resumed_fan_out_streak_exactly_like_a_single_call_streak() -> None: + detector = monitor() + for round_index in (0, 1): + await detector.declare_round(round_index, read_calls(["a", "b"])) + for path in ("a", "b"): + await detector.record_tool_call("read", {"path": path}, round_index) + + # JSON round-trip: the set must survive real serialization. + wire = json.loads(json.dumps(detector.get_state())) + resumed = DoomLoopMonitor(resolve_doom_loop_option(True), wire) + await resumed.declare_round(0, read_calls(["a", "b"])) + record = await resumed.record_tool_call("read", {"path": "a"}, 0) + + assert record["streak"] == 3 + assert _action(record) == "block" + + +async def test_keeps_counting_a_repeat_when_a_paused_hitl_member_drops_from_the_resumed_round() -> None: + detector = monitor() + work_streaks: List[str] = [] + for round_index in (0, 1, 2): + members = ["work", "gated"] if round_index == 0 else ["work"] + await detector.declare_round(round_index, read_calls(members, "deploy")) + for path in members: + record = await detector.record_tool_call("deploy", {"path": path}, round_index) + if path == "work": + work_streaks.append(f"{record['streak']}:{_action(record)}") + assert work_streaks == ["1:none", "2:observe", "3:block"] + + +async def test_collapses_staggered_per_call_counts_in_one_round_to_a_single_message() -> None: + detector = steer_monitor() + rounds = [["a"], ["a", "b"], ["a", "b", "c"], ["a", "b", "c", "d"]] + messages: List[str] = [] + streaks: List[int] = [] + for round_index, paths in enumerate(rounds): + await detector.declare_round(round_index, read_calls(paths)) + for path in paths: + record = await detector.record_tool_call("read", {"path": path}, round_index) + verdict = record.get("verdict") + if round_index == 3 and verdict is not None: + messages.append(verdict["message"]) + streaks.append(verdict["streak"]) + + # Three verdicts with distinct counts -- the payload keeps the numbers... + assert streaks == [4, 3, 2] + # ...but the prose is byte-identical, so steer queues ONE correction. + assert len(messages) == 3 + assert len(set(messages)) == 1 + + +async def test_bounds_a_mixed_evidence_round_to_one_message_per_distinct_fact() -> None: + detector = steer_monitor() + rounds = [["a"], ["a", "b"], ["a", "b"]] + last_round = len(rounds) - 1 + messages: List[str] = [] + for round_index, paths in enumerate(rounds): + await detector.declare_round(round_index, read_calls(paths)) + for path in paths: + record = await detector.record_tool_call("read", {"path": path}, round_index) + verdict = record.get("verdict") + if round_index == last_round and verdict is not None: + messages.append(verdict["message"]) + + assert len(messages) == 2 + assert len(set(messages)) == 2 + assert 'this exact "read" call has been repeated' in messages[0] + assert "same set of 2 parallel calls" in messages[1] + + +async def test_collapses_per_call_steer_messages_across_a_wide_repeating_round() -> None: + detector = steer_monitor() + wide = [f"f{index}" for index in range(20)] + messages = set() + verdict_count = 0 + for round_index, paths in enumerate([wide, wide, [*wide, "new"]]): + await detector.declare_round(round_index, read_calls(paths)) + for path in paths: + record = await detector.record_tool_call("read", {"path": path}, round_index) + verdict = record.get("verdict") + if round_index == 2 and verdict is not None: + verdict_count += 1 + messages.add(verdict["message"]) + + # All 20 repeated members fire; the steer queue sees ONE correction. + assert verdict_count == 20 + assert len(messages) == 1 + + +async def test_persists_per_call_evidence_when_the_round_has_shrunk_to_a_single_call() -> None: + detector = monitor() + await detector.declare_round(0, read_calls(["work", "gated"], "deploy")) + await detector.record_tool_call("deploy", {"path": "work"}, 0) + await detector.record_tool_call("deploy", {"path": "gated"}, 0) + # The gated call paused; the resumed round is just the working call. + await detector.declare_round(1, read_calls(["work"], "deploy")) + before_save = await detector.record_tool_call("deploy", {"path": "work"}, 1) + assert before_save["streak"] == 2 + + wire = json.loads(json.dumps(detector.get_state())) + resumed = DoomLoopMonitor(resolve_doom_loop_option(True), wire) + await resumed.declare_round(0, read_calls(["work"], "deploy")) + after_resume = await resumed.record_tool_call("deploy", {"path": "work"}, 0) + assert after_resume["streak"] == 3 + assert _action(after_resume) == "block" + + +async def test_scores_a_resumed_single_call_on_its_own_earned_evidence_never_inherited() -> None: + detector = monitor() + paths = ["a", "b", "c"] + for round_index in (0, 1): + await detector.declare_round(round_index, read_calls(paths)) + for path in paths: + await detector.record_tool_call("read", {"path": path}, round_index) + + saved = detector.get_state() + read_state = saved["tools"]["read"] + assert read_state["streak"] == 2 + assert len(read_state["round_fingerprints"]) == 3 + # Steady state: per-call counts equal the round streak, so they are omitted. + assert "call_streaks" not in read_state + + for path in paths: + resumed = DoomLoopMonitor(resolve_doom_loop_option(True), detector.get_state()) + await resumed.declare_round(0, read_calls([path])) + solo = await resumed.record_tool_call("read", {"path": path}, 0) + assert solo["streak"] == 3 + assert _action(solo) == "block" + + resumed_fresh = DoomLoopMonitor(resolve_doom_loop_option(True), detector.get_state()) + await resumed_fresh.declare_round(0, read_calls(["never-before"])) + fresh = await resumed_fresh.record_tool_call("read", {"path": "never-before"}, 0) + assert fresh["streak"] == 1 + assert "verdict" not in fresh + + +async def test_renders_one_verdict_text_per_undeclared_multi_call_round() -> None: + detector = DoomLoopMonitor(resolve_doom_loop_option(True)) + verdict_rounds = 0 + for round_index in range(1, 5): + texts: List[str] = [] + for cmd in ("a", "b"): + result = await detector.record_tool_call("sh", {"cmd": cmd}, round_index, allow_block=False) + verdict = result.get("verdict") + if verdict is not None: + texts.append(verdict["message"]) + # Either no verdict yet, or every member produced byte-identical text. + assert len(set(texts)) <= 1 + if texts: + verdict_rounds += 1 + assert len(texts) == 2 + # Rounds 2-4 cross observe (2) and the non-blockable block rung (3). + assert verdict_rounds == 3 + + +async def test_keeps_the_fingerprint_bearing_text_for_a_genuine_single_call_round() -> None: + detector = DoomLoopMonitor(resolve_doom_loop_option(True)) + last = None + for round_index in range(1, 4): + await detector.declare_round(round_index, read_calls(["same"])) + result = await detector.record_tool_call("read", {"path": "same"}, round_index) + verdict = result.get("verdict") + last = verdict["message"] if verdict is not None else last + assert last is not None + assert "identical arguments (fingerprint" in last diff --git a/tests/unit/test_doom_loop_integration.py b/tests/unit/test_doom_loop_integration.py new file mode 100644 index 0000000..745463c --- /dev/null +++ b/tests/unit/test_doom_loop_integration.py @@ -0,0 +1,793 @@ +"""Integration tests for doom-loop detection in the call_model tool loop. + +Ports `packages/agent/tests/unit/doom-loop-integration.test.ts`. + +No live model anywhere: the transport is a `QueuedClient` playing scripted +responses that SIMULATE an LLM stuck in a doom loop -- the same tool call with +identical arguments turn after turn, repeated EMPTY tool calls, repeated +unparseable (invalid-JSON) calls, and degenerate repeated-token text. Detection +is deterministic, so each scenario asserts the exact turn on which the ladder +fires and exactly how the engine responds (hook emission, blocked tool output, +halted loop, persisted state). + +Port notes: +- Hook payloads are snake_case (`tool_name`, `tool_input`), per hooks_schemas.py. +- Outgoing `function_call_output` items carry snake_case `call_id`: + `ModelResult._send` normalizes at the transport boundary. Deliberate. +- Python has no `undefined`: a `loop_key` callable returning `None` exempts the + call (upstream's `null`), see doom_loop.py's module docstring. +""" + +from __future__ import annotations + +import json +import warnings +from typing import Any, Dict, Iterator, List, Mapping + +import pytest + +from openrouter_agent import HookEntry, HooksManager, call_model, resolve_loop_key_material, tool +from tests._fixtures import MemoryStateAccessor, QueuedClient, function_call_item, make_response, message_item + +#: Expected detector diagnostics (ladder hazards, unhashable key material) are +#: asserted where they matter; silence them elsewhere. +pytestmark = pytest.mark.filterwarnings("ignore:\\[DoomLoop\\]:UserWarning") + +# --------------------------------------------------------------------------- +# Scripted-response fixtures (the "simulated LLM"), built on tests/_fixtures.py +# --------------------------------------------------------------------------- + + +class Script: + """Numbers responses `resp_N` and calls `call_N` like upstream's counters.""" + + def __init__(self) -> None: + self.responses = 0 + self.calls = 0 + + def response(self, output: List[Dict[str, Any]]) -> Dict[str, Any]: + self.responses += 1 + return make_response(f"resp_{self.responses}", output) + + def call(self, name: str, args: Any) -> Dict[str, Any]: + self.calls += 1 + return function_call_item(f"call_{self.calls}", name, json.dumps(args)) + + def message(self, text: str) -> Dict[str, Any]: + self.calls += 1 + return message_item(text, item_id=f"msg_{self.calls}") + + def tool_call_turn(self, name: str, args: Any) -> Dict[str, Any]: + return self.response([self.call(name, args)]) + + def invalid_json_turn(self, name: str, raw_arguments: str) -> Dict[str, Any]: + self.calls += 1 + return self.response([function_call_item(f"call_{self.calls}", name, raw_arguments)]) + + def text_turn(self, text: str) -> Dict[str, Any]: + return self.response([self.message(text)]) + + def fan_out_turn(self, name: str, args_list: List[Any]) -> Dict[str, Any]: + return self.response([self.call(name, args) for args in args_list]) + + +def sent_tool_outputs(client: QueuedClient, request_index: int) -> List[Mapping[str, Any]]: + """The function_call_output items the engine sent in request `request_index`.""" + request = client.requests[request_index] + items = request.get("input") if isinstance(request.get("input"), list) else [] + return [item for item in items or [] if isinstance(item, Mapping) and item.get("type") == "function_call_output"] + + +def collect(hooks: HooksManager, hook_name: str) -> List[Dict[str, Any]]: + collected: List[Dict[str, Any]] = [] + + def handler(payload: Dict[str, Any], ctx: Any) -> None: + collected.append(payload) + + hooks.on(hook_name, HookEntry(handler=handler)) + return collected + + +def actions_and_streaks(detections: List[Dict[str, Any]]) -> List[List[Any]]: + return [[d["action"], d["streak"]] for d in detections] + + +class Counter: + def __init__(self, result: Any) -> None: + self.count = 0 + self.result = result + + def __call__(self, params: Any, ctx: Any = None) -> Any: + self.count += 1 + return self.result + + +def _search_execute(params: Any, ctx: Any) -> Dict[str, Any]: + return {"results": [f"result for {params['query']}"]} + + +# Doom-loop identity: the query only -- "a web-search tool hashes its query". +search_tool = tool( + name="web_search", + description="Search the web.", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: args["query"].strip().lower(), + execute=_search_execute, +) + + +@pytest.fixture() +def script() -> Iterator[Script]: + yield Script() + + +# --------------------------------------------------------------------------- +# Scenario 1: identical tool calls, turn after turn +# --------------------------------------------------------------------------- + + +async def test_fan_out_escalates_observe_to_block_through_call_model_and_stops_the_whole_fan_out( + script: Script, +) -> None: + executed: List[str] = [] + + def read(params: Any, ctx: Any) -> Dict[str, Any]: + executed.append(params["path"]) + return {"content": f"contents of {params['path']}"} + + read_tool = tool(name="read", input_schema=dict, output_schema=dict, execute=read) + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + + paths = [{"path": "a"}, {"path": "b"}, {"path": "c"}] + client = QueuedClient( + [ + script.fan_out_turn("read", paths), + script.fan_out_turn("read", paths), + script.fan_out_turn("read", paths), + script.fan_out_turn("read", paths), + script.text_turn("recovered"), + ] + ) + + text = await call_model( + client, + { + "model": "test-model", + "input": "Summarize these files.", + "tools": [read_tool], + "doom_loop": True, + "hooks": hooks, + }, + ).get_text() + + assert text == "recovered" + # Round 2 observes all three calls; rounds 3-4 block all three. The block + # rung stops the WHOLE fan-out spending: only rounds 1-2 execute. + assert actions_and_streaks(detections) == [ + ["observe", 2], + ["observe", 2], + ["observe", 2], + ["block", 3], + ["block", 3], + ["block", 3], + ["block", 4], + ["block", 4], + ["block", 4], + ] + assert executed == ["a", "b", "c", "a", "b", "c"] + assert len(client.requests) == 5 + # Round verdicts (not per-call) name the set: pins the round declaration. + for detection in detections: + assert "same set of 3 parallel calls" in detection["message"] + + +async def test_a_repeat_sharing_its_id_with_a_varying_sibling_is_still_detected(script: Script) -> None: + read_tool = tool(name="read", input_schema=dict, output_schema=dict, execute=lambda p, c: {"content": p["path"]}) + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + + def dup_id_turn(round_index: int) -> Dict[str, Any]: + items = [] + for index, args in enumerate([{"path": "a"}, {"path": f"b{round_index}"}]): + item = function_call_item("call_X", "read", json.dumps(args)) + item["id"] = f"fc_{round_index}_{index}" + items.append(item) + return script.response(items) + + client = QueuedClient([dup_id_turn(0), dup_id_turn(1), dup_id_turn(2), dup_id_turn(3), script.text_turn("done")]) + + try: + await call_model( + client, {"model": "test-model", "input": "go", "tools": [read_tool], "doom_loop": True, "hooks": hooks} + ).get_text() + except Exception: # noqa: BLE001 - upstream `.catch(() => undefined)`: only detection is asserted + pass + + # The repeated `a` accumulates despite the id collision (upstream asserts + # membership of 'observe' and 'block'; the exact sequence is pinned here): + # only `a` repeats, so it observes at 2 and blocks at 3 and 4. + assert [d["action"] for d in detections] == ["observe", "block", "block"] + assert all(d["tool_input"] == {"path": "a"} for d in detections) + + +async def test_observes_at_2_blocks_at_3_with_an_explanatory_tool_error_and_lets_the_model_recover( + script: Script, +) -> None: + execute = Counter(None) + + def spied(params: Any, ctx: Any) -> Dict[str, Any]: + execute(params) + return {"results": [f"result for {params['query']}"]} + + spied_search = tool( + name="web_search", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: args["query"].strip().lower(), + execute=spied, + ) + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + + # Three identical searches (case/whitespace vary -- loop_key collapses them). + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "openrouter sdk"}), + script.tool_call_turn("web_search", {"query": "OpenRouter SDK"}), + script.tool_call_turn("web_search", {"query": " openrouter sdk "}), + script.text_turn("Giving up on repeating; here is my answer."), + ] + ) + + text = await call_model( + client, + {"model": "test-model", "input": "Find the SDK.", "tools": [spied_search], "doom_loop": True, "hooks": hooks}, + ).get_text() + + assert text == "Giving up on repeating; here is my answer." + # Calls 1 and 2 executed; call 3 was BLOCKED before execution. + assert execute.count == 2 + assert actions_and_streaks(detections) == [["observe", 2], ["block", 3]] + assert detections[0]["detector"] == "tool-fingerprint" + assert detections[0]["tool_name"] == "web_search" + + # Request 4 carries the third call's (blocked) function_call_output. + outputs = sent_tool_outputs(client, 3) + blocked = outputs[-1] + assert blocked["call_id"] == "call_3" + assert "Doom loop suspected" in blocked["output"] + assert "web_search" in blocked["output"] + assert "3 consecutive rounds" in blocked["output"] + + +async def test_stops_the_run_at_the_stop_threshold_instead_of_spending_forever(script: Script) -> None: + client = QueuedClient([script.tool_call_turn("web_search", {"query": "same query"}) for _ in range(10)]) + + result = call_model( + client, + { + "model": "test-model", + "input": "Find it.", + "tools": [search_tool], + "doom_loop": {"ladder": {"observe": 2, "block": 3, "stop": 4}}, + }, + ) + response = await result.get_response() + + # Streaks: 1 (executes), 2 (observe), 3 (block), 4 (stop). The loop halts + # before requesting a 5th turn. + assert len(client.requests) == 4 + verdict = await result.get_doom_loop_verdict() + assert verdict is not None + assert verdict["action"] == "stop" + assert verdict["detector"] == "tool-fingerprint" + assert verdict["streak"] == 4 + assert verdict["tool_name"] == "web_search" + # The run surfaces the last response rather than throwing. + assert response["id"] == "resp_4" + + +async def test_emits_session_end_with_reason_doom_loop_when_stopped(script: Script) -> None: + hooks = HooksManager() + session_ends = collect(hooks, "SessionEnd") + client = QueuedClient([script.tool_call_turn("web_search", {"query": "x"}) for _ in range(6)]) + + await call_model( + client, + { + "model": "test-model", + "input": "go", + "tools": [search_tool], + "doom_loop": {"ladder": {"stop": 3}}, + "hooks": hooks, + }, + ).get_response() + + assert len(session_ends) == 1 + assert session_ends[0]["reason"] == "doom_loop" + assert len(client.requests) == 3 + + +async def test_does_not_interfere_with_a_healthy_run(script: Script) -> None: + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "first"}), + script.tool_call_turn("web_search", {"query": "second"}), + script.tool_call_turn("web_search", {"query": "third"}), + script.text_turn("All three found."), + ] + ) + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + + text = await call_model( + client, + { + "model": "test-model", + "input": "Search three things.", + "tools": [search_tool], + "doom_loop": True, + "hooks": hooks, + }, + ).get_text() + + assert text == "All three found." + assert detections == [] + assert len(client.requests) == 4 + + +async def test_is_fully_inert_when_doom_loop_is_not_configured(script: Script) -> None: + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.text_turn("done"), + ] + ) + accessor = MemoryStateAccessor() + + text = await call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "state": accessor} + ).get_text() + + # All four identical calls executed; nothing blocked, nothing stopped. + assert text == "done" + for request_index in (1, 2, 3, 4): + outputs = sent_tool_outputs(client, request_index) + assert "Doom loop" not in outputs[-1]["output"] + # And nothing is persisted for a detector that is off. + assert accessor.stored is not None + assert accessor.stored.doom_loop is None + + +# --------------------------------------------------------------------------- +# Scenario 2: repeated EMPTY tool calls +# --------------------------------------------------------------------------- + + +async def test_treats_repeated_empty_calls_as_a_streak_and_blocks_at_the_threshold(script: Script) -> None: + list_tool = tool( + name="list_tasks", + description="List everything.", + input_schema=dict, + output_schema=dict, + execute=lambda p, c: {"tasks": ["task-1"]}, + ) + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + client = QueuedClient( + [ + script.tool_call_turn("list_tasks", {}), + script.tool_call_turn("list_tasks", {}), + script.tool_call_turn("list_tasks", {}), + script.text_turn("The list has not changed; stopping."), + ] + ) + + text = await call_model( + client, + {"model": "test-model", "input": "What tasks exist?", "tools": [list_tool], "doom_loop": True, "hooks": hooks}, + ).get_text() + + assert text == "The list has not changed; stopping." + assert actions_and_streaks(detections) == [["observe", 2], ["block", 3]] + assert "Doom loop suspected" in sent_tool_outputs(client, 3)[-1]["output"] + + +# --------------------------------------------------------------------------- +# Scenario 3: repeated malformed (invalid JSON) tool calls +# --------------------------------------------------------------------------- + + +async def test_counts_parse_error_repeats_and_stops_instead_of_bouncing_forever(script: Script) -> None: + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + client = QueuedClient([script.invalid_json_turn("web_search", '{"query": broken') for _ in range(4)]) + + result = call_model( + client, + { + "model": "test-model", + "input": "Search.", + "tools": [search_tool], + "doom_loop": {"ladder": {"observe": 2, "block": 3, "stop": 3}}, + "hooks": hooks, + }, + ) + await result.get_response() + + # Streak 2 observes; streak 3 crosses stop (stop wins over block). + assert actions_and_streaks(detections) == [["observe", 2], ["stop", 3]] + # Raw-string arguments are not a record: no tool_input on the payload. + assert all(d["tool_input"] is None for d in detections) + # Halted after the 3rd scripted turn -- the 4th was never requested. + assert len(client.requests) == 3 + verdict = await result.get_doom_loop_verdict() + assert verdict is not None + assert (verdict["action"], verdict["streak"]) == ("stop", 3) + + +# --------------------------------------------------------------------------- +# Scenario 4: doom-loop TEXT tokens +# --------------------------------------------------------------------------- + + +async def test_stops_a_run_whose_response_is_one_degenerate_repeated_token_loop(script: Script) -> None: + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + doom_text = " ".join(["I need to check the file again."] * 40) + client = QueuedClient([script.text_turn(doom_text)]) + + result = call_model(client, {"model": "test-model", "input": "Summarize.", "doom_loop": True, "hooks": hooks}) + text = await result.get_text() + + # The tokens were already emitted -- the response is surfaced as-is -- but + # the verdict fired deterministically with the repeat count. + assert text == doom_text + assert len(detections) == 1 + assert detections[0]["detector"] == "text-repetition" + assert detections[0]["action"] == "stop" + assert detections[0]["streak"] == 40 + + +async def test_halts_a_tool_loop_when_every_turn_repeats_identical_filler_text(script: Script) -> None: + filler = "Hmm, let me look somewhere else." + + def text_and_call_turn(query: str) -> Dict[str, Any]: + return script.response([script.message(filler), script.call("web_search", {"query": query})]) + + client = QueuedClient([text_and_call_turn(f"q{i}") for i in range(1, 6)]) + + result = call_model( + client, + { + "model": "test-model", + "input": "Find it.", + "tools": [search_tool], + "doom_loop": {"ladder": {"observe": 2, "block": 3, "stop": 3}, "text": True}, + }, + ) + await result.get_response() + + # Text streak: turn1=1, turn2=2 (observe), turn3=3 (stop): exactly 3 requests. + assert len(client.requests) == 3 + verdict = await result.get_doom_loop_verdict() + assert verdict is not None + assert (verdict["detector"], verdict["action"], verdict["streak"]) == ("text-streak", "stop", 3) + + +# --------------------------------------------------------------------------- +# loop_key contract +# --------------------------------------------------------------------------- + + +async def test_bash_style_loop_key_same_command_in_a_different_cwd_is_not_a_loop(script: Script) -> None: + runs: List[str] = [] + + def run(params: Any, ctx: Any) -> Dict[str, Any]: + runs.append(f"{params['cwd']}:{params['command']}") + return {"stdout": f"ran {params['command']} in {params['cwd']}"} + + bash_tool = tool( + name="bash", + description="Run a command.", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: {"command": args["command"].strip(), "cwd": args["cwd"]}, + execute=run, + ) + client = QueuedClient( + [ + script.tool_call_turn("bash", {"command": "ls", "cwd": "/a"}), + script.tool_call_turn("bash", {"command": "ls", "cwd": "/b"}), + script.tool_call_turn("bash", {"command": "ls", "cwd": "/c"}), + script.text_turn("Directory listings acquired."), + ] + ) + + text = await call_model( + client, + { + "model": "test-model", + "input": "List all three directories.", + "tools": [bash_tool], + "doom_loop": {"ladder": {"block": 2}}, # aggressive: ANY repeat would block + }, + ).get_text() + + assert text == "Directory listings acquired." + assert runs == ["/a:ls", "/b:ls", "/c:ls"] + + +async def test_loop_key_returning_none_exempts_a_legitimately_repetitive_tool(script: Script) -> None: + poll_tool = tool( + name="check_status", + description="Poll job status.", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: None, # repetition is this tool's job + execute=lambda p, c: {"done": False}, + ) + client = QueuedClient( + [ + *[script.tool_call_turn("check_status", {"jobId": "job-1"}) for _ in range(5)], + script.text_turn("Job still running after 5 polls."), + ] + ) + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + + text = await call_model( + client, + {"model": "test-model", "input": "Wait for the job.", "tools": [poll_tool], "doom_loop": True, "hooks": hooks}, + ).get_text() + + assert text == "Job still running after 5 polls." + assert detections == [] + assert len(client.requests) == 6 + + +async def test_a_raising_loop_key_falls_back_to_full_arguments_instead_of_failing_the_run(script: Script) -> None: + def broken_key(args: Any) -> Any: + raise RuntimeError("loopKey bug") + + broken_key_tool = tool( + name="flaky", input_schema=dict, output_schema=dict, loop_key=broken_key, execute=lambda p, c: {"ok": True} + ) + client = QueuedClient( + [ + script.tool_call_turn("flaky", {"value": "same"}), + script.tool_call_turn("flaky", {"value": "same"}), + script.tool_call_turn("flaky", {"value": "same"}), + script.text_turn("done"), + ] + ) + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + text = await call_model( + client, + {"model": "test-model", "input": "go", "tools": [broken_key_tool], "doom_loop": True, "hooks": hooks}, + ).get_text() + + # Detection still worked via the fallback identity. + assert text == "done" + assert actions_and_streaks(detections) == [["observe", 2], ["block", 3]] + fallback_warnings = [ + str(w.message) + for w in caught + if "loop_key threw (loopKey bug); falling back to full arguments" in str(w.message) + ] + # One per checkpointed call (the round declaration does not warn). + assert len(fallback_warnings) == 3 + + +async def test_resolve_loop_key_material_can_raise_so_its_callers_must_guard_it() -> None: + """The field-list form reads the arguments (`field in args`, `args[field]`), + so a hostile mapping escapes it uncaught. Pinned so a refactor cannot + quietly remove the guards in `_begin_doom_loop_round` and the checkpoint. + Python analog of upstream's throwing getter / proxy `has` trap.""" + + class ThrowingGetter(Dict[str, Any]): + def __getitem__(self, key: str) -> Any: + raise RuntimeError("getter boom") + + throwing_getter = ThrowingGetter(command="ls") + with pytest.raises(RuntimeError, match="getter boom"): + resolve_loop_key_material(["command"], throwing_getter) + + class HostileContains(Dict[str, Any]): + def __contains__(self, key: object) -> bool: + raise RuntimeError("has boom") + + with pytest.raises(RuntimeError, match="has boom"): + resolve_loop_key_material(["command"], HostileContains()) + + +# --------------------------------------------------------------------------- +# DoomLoopDetected hook override_action +# --------------------------------------------------------------------------- + + +async def test_a_handler_can_de_escalate_a_block_to_observe(script: Script) -> None: + execute = Counter({"ok": True}) + noisy = tool(name="noisy", input_schema=dict, output_schema=dict, execute=execute) + hooks = HooksManager() + hooks.on("DoomLoopDetected", HookEntry(handler=lambda p, c: {"override_action": "observe"})) + client = QueuedClient( + [ + script.tool_call_turn("noisy", {}), + script.tool_call_turn("noisy", {}), + script.tool_call_turn("noisy", {}), + script.text_turn("done"), + ] + ) + + text = await call_model( + client, {"model": "test-model", "input": "go", "tools": [noisy], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert text == "done" + # Without the override the 3rd call would be blocked. + assert execute.count == 3 + + +async def test_a_handler_can_escalate_an_observe_to_stop(script: Script) -> None: + hooks = HooksManager() + hooks.on("DoomLoopDetected", HookEntry(handler=lambda p, c: {"override_action": "stop"})) + client = QueuedClient([script.tool_call_turn("web_search", {"query": "x"}) for _ in range(3)]) + + result = call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "doom_loop": True, "hooks": hooks} + ) + await result.get_response() + + # Streak 2 (observe) escalated to stop: the loop halts after the 2nd turn. + assert len(client.requests) == 2 + verdict = await result.get_doom_loop_verdict() + assert verdict is not None + assert verdict["streak"] == 2 + + +# --------------------------------------------------------------------------- +# State persistence: a resumed doom loop is still a doom loop +# --------------------------------------------------------------------------- + + +async def test_persists_streaks_into_conversation_state_and_keeps_counting_across_a_resume(script: Script) -> None: + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "persistent"}), + script.tool_call_turn("web_search", {"query": "persistent"}), + script.text_turn("pausing here"), + ] + ) + await call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "doom_loop": True, "state": accessor} + ).get_text() + + assert accessor.stored is not None + assert accessor.stored.doom_loop is not None + assert accessor.stored.doom_loop["tools"]["web_search"]["streak"] == 2 + + # Run 2: ONE more identical call crosses the block rung -- the streak + # survived the round-trip. + hooks = HooksManager() + detections = collect(hooks, "DoomLoopDetected") + client2 = QueuedClient( + [script.tool_call_turn("web_search", {"query": "persistent"}), script.text_turn("recovered")] + ) + await call_model( + client2, + { + "model": "test-model", + "input": "continue", + "tools": [search_tool], + "doom_loop": True, + "state": accessor, + "hooks": hooks, + }, + ).get_text() + + assert actions_and_streaks(detections) == [["block", 3]] + + +async def test_doom_loop_state_round_trips_through_json_serialization(script: Script) -> None: + from openrouter_agent import deserialize_conversation_state, serialize_conversation_state + + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "x"}), + script.tool_call_turn("web_search", {"query": "x"}), + script.text_turn("done"), + ] + ) + await call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "doom_loop": True, "state": accessor} + ).get_text() + + state = accessor.stored + assert state is not None + assert state.doom_loop is not None + # Plain JSON: survives dumps/loads identically, and through the versioned + # serialization contract. + assert json.loads(json.dumps(state.doom_loop)) == state.doom_loop + assert deserialize_conversation_state(serialize_conversation_state(state)).doom_loop == state.doom_loop + + +async def test_invokes_a_tool_supplied_loop_key_exactly_once_per_call(script: Script) -> None: + calls: List[str] = [] + + def loop_key(args: Any) -> str: + calls.append(args["query"]) + return str(args["query"]).strip().lower() + + counted_search = tool( + name="web_search", input_schema=dict, output_schema=dict, loop_key=loop_key, execute=_search_execute + ) + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "first"}), + script.tool_call_turn("web_search", {"query": "second"}), + script.text_turn("Done."), + ] + ) + + await call_model( + client, {"model": "test-model", "input": "Search twice.", "tools": [counted_search], "doom_loop": True} + ).get_text() + + # Two calls issued, so exactly two invocations -- not four. + assert calls == ["first", "second"] + + +async def test_never_invokes_loop_key_for_a_manual_call_the_detector_will_not_check(script: Script) -> None: + manual_calls: List[Any] = [] + executable_calls: List[Any] = [] + + def manual_key(args: Any) -> Any: + manual_calls.append(args) + return args["query"] + + def executable_key(args: Any) -> Any: + executable_calls.append(args) + return args["query"] + + manual_tool = tool(name="ask_human", input_schema=dict, output_schema=dict, loop_key=manual_key, execute=False) + executable_tool = tool( + name="web_search", + input_schema=dict, + output_schema=dict, + loop_key=executable_key, + execute=lambda p, c: {"results": [p["query"]]}, + ) + # MIXED batch: an all-manual round never reaches the doom-loop round. + client = QueuedClient( + [ + script.response( + [script.call("web_search", {"query": "go"}), script.call("ask_human", {"query": "what now?"})] + ) + ] + ) + + await call_model( + client, + { + "model": "test-model", + "input": "Search, and ask me something.", + "tools": [executable_tool, manual_tool], + "doom_loop": True, + }, + ).get_text() + + assert len(executable_calls) == 1 + assert manual_calls == [] + assert len(client.requests) == 1 diff --git a/tests/unit/test_doom_loop_public_api.py b/tests/unit/test_doom_loop_public_api.py new file mode 100644 index 0000000..3a18e1d --- /dev/null +++ b/tests/unit/test_doom_loop_public_api.py @@ -0,0 +1,134 @@ +"""Consumer-facing contract for driving `DoomLoopMonitor` directly. + +Port of tests/unit/doom-loop-public-api.test.ts. Upstream imports from the +package entrypoint; until `openrouter_agent/__init__.py` re-exports the +doom-loop symbols this imports from `openrouter_agent.doom_loop`. Once the +re-export lands, switch the import to `openrouter_agent` so the test pins the +public surface again. +""" + +from __future__ import annotations + +import json +from typing import Any, List, Optional + +from openrouter_agent.doom_loop import ( + DEFAULT_DOOM_LOOP_LADDER, + DoomLoopMonitor, + DoomLoopRoundCall, + resolve_doom_loop_option, +) + + +def _read_calls(paths: List[str]) -> List[DoomLoopRoundCall]: + return [{"tool_name": "read", "key_material": {"path": path}} for path in paths] + + +async def test_constructs_with_defaults_detects_a_repeating_fan_out_and_honors_the_exported_ladder() -> None: + monitor = DoomLoopMonitor(resolve_doom_loop_option(True)) + paths = ["a", "b", "c"] + + verdict_action: Optional[str] = None + verdict_streak: Optional[int] = None + for round_number in (0, 1, 2): + await monitor.declare_round(round_number, _read_calls(paths)) + for path in paths: + record = await monitor.record_tool_call("read", {"path": path}, round_number) + verdict = record.get("verdict") + if verdict is not None: + verdict_action = verdict["action"] + verdict_streak = verdict["streak"] + + # Third identical round crosses the exported default block threshold. + assert verdict_streak == DEFAULT_DOOM_LOOP_LADDER["block"] + assert verdict_action == "block" + + +async def test_accepts_a_config_object_and_honors_a_custom_ladder() -> None: + monitor = DoomLoopMonitor(resolve_doom_loop_option({"ladder": {"observe": 2, "block": False, "stop": 4}})) + + actions: List[Optional[str]] = [] + for round_number in (0, 1, 2, 3): + record = await monitor.record_tool_call("search", {"q": "same"}, round_number) + verdict = record.get("verdict") + actions.append(verdict["action"] if verdict else None) + # block disabled; the streak of 4 reaches the custom stop rung. + assert actions[-1] == "stop" + assert actions == [None, "observe", "observe", "stop"] + + +async def test_accumulates_a_fan_out_streak_across_per_turn_process_boundaries() -> None: + # The serverless pattern: one round per call_model run, state persisted + # between turns. A repeating fan-out must accumulate across boundaries — + # this is why the round's fingerprint SET is persisted. + paths = ["a", "b", "c"] + wire: Any = None + per_turn: List[str] = [] + for _turn in range(4): + monitor = DoomLoopMonitor(resolve_doom_loop_option(True), wire) + await monitor.declare_round(0, _read_calls(paths)) + last = "" + for path in paths: + record = await monitor.record_tool_call("read", {"path": path}, 0) + verdict = record.get("verdict") + last = f"{record['streak']}:{verdict['action'] if verdict else 'none'}" + per_turn.append(last) + wire = json.loads(json.dumps(monitor.get_state())) + + assert per_turn == ["1:none", "2:observe", "3:block", "4:block"] + + +async def test_restores_a_legacy_blob_without_round_fingerprints_with_single_call_semantics() -> None: + legacy = {"tools": {"read": {"fingerprint": "not-a-real-fingerprint", "streak": 2}}} + monitor = DoomLoopMonitor(resolve_doom_loop_option(True), legacy) + different = await monitor.record_tool_call("read", {"path": "x"}, 0) + assert different["streak"] == 1 + + hostile = {"tools": {"read": {"fingerprint": "ab", "streak": 2, "round_fingerprints": [1, {}, None]}}} + survives = DoomLoopMonitor(resolve_doom_loop_option(True), hostile) + record = await survives.record_tool_call("read", {"path": "x"}, 0) + assert record["streak"] == 1 + # The malformed set degraded to the lone fingerprint rather than dropping the entry. + assert survives.get_state()["tools"]["read"]["streak"] == 1 + + +async def test_isolates_the_live_detector_from_mutations_of_a_saved_snapshot() -> None: + paths = ["a", "b"] + monitor = DoomLoopMonitor(resolve_doom_loop_option(True)) + for round_number in (0, 1): + await monitor.declare_round(round_number, _read_calls(paths)) + for path in paths: + await monitor.record_tool_call("read", {"path": path}, round_number) + + # A careless (or hostile) caller mutates the saved snapshot in place. + saved = monitor.get_state() + round_fingerprints = saved["tools"]["read"].get("round_fingerprints") + assert round_fingerprints is not None + assert len(round_fingerprints) == 2 + round_fingerprints.append("INJECTED") + + # The live detector must be unaffected: round 3 still blocks. + await monitor.declare_round(2, _read_calls(paths)) + last_streak = 0 + last_action: Optional[str] = None + for path in paths: + record = await monitor.record_tool_call("read", {"path": path}, 2) + verdict = record.get("verdict") + last_streak = record["streak"] + last_action = verdict["action"] if verdict else None + assert last_streak == 3 + assert last_action == "block" + + +async def test_round_trips_state_across_a_process_boundary_via_plain_json() -> None: + first = DoomLoopMonitor(resolve_doom_loop_option(True)) + await first.record_tool_call("search", {"q": "same"}, 0) + await first.record_tool_call("search", {"q": "same"}, 1) + + # Simulate the serverless pattern: serialize, "new process", restore. + wire = json.dumps(first.get_state()) + second = DoomLoopMonitor(resolve_doom_loop_option(True), json.loads(wire)) + resumed = await second.record_tool_call("search", {"q": "same"}, 0) + + # Single-call streak continues across the boundary: 2 -> 3. + assert resumed["streak"] == 3 diff --git a/tests/unit/test_doom_loop_remediation.py b/tests/unit/test_doom_loop_remediation.py new file mode 100644 index 0000000..01e41a4 --- /dev/null +++ b/tests/unit/test_doom_loop_remediation.py @@ -0,0 +1,983 @@ +"""Integration tests for the doom-loop remediation pass. + +Ports `packages/agent/tests/unit/doom-loop-remediation.test.ts`. Every +scenario simulates the LLM with scripted `QueuedClient` responses. + +Coverage map (finding -> test group): +- B1/D7 state integrity + session reason on text-verdict stops +- B3 parallel same-round duplicates are one piece of evidence +- D1 no final-response request after a doom stop +- D2/D3 approval-resume gating + verdict persistence/clearing +- D4 steer survives pauses via persisted pending_steer; in-loop steer +- D5 loop_key returning nothing -> no colliding constant identity +- H1 DOCUMENTED MISS: nonce-varying args evade the fingerprint +- H2 server-tool repetition detected at the step checkpoint +- API1 loop_key forms (computed function, False) +- config ON->OFF / ON->ON lifecycle across resumes + +Port notes: state/hook keys are snake_case (`pending_steer`, `stop_verdict`, +`consumed_forced_tool_choice_key`); request input items carry `call_id` +(normalized at the transport boundary by `ModelResult._send`), while persisted +state messages keep upstream's `callId`. +""" + +from __future__ import annotations + +import dataclasses +import json +import warnings +from typing import Any, Dict, Iterator, List, Mapping + +import pytest + +from openrouter_agent import HookEntry, HooksManager, call_model, tool +from tests._fixtures import MemoryStateAccessor, QueuedClient, function_call_item, make_response, message_item + +pytestmark = pytest.mark.filterwarnings("ignore:\\[DoomLoop\\]:UserWarning") + + +class Script: + def __init__(self) -> None: + self.responses = 0 + self.calls = 0 + + def turn(self, *items: Dict[str, Any]) -> Dict[str, Any]: + self.responses += 1 + return make_response(f"resp_{self.responses}", list(items)) + + def call(self, name: str, args: Any) -> Dict[str, Any]: + self.calls += 1 + return function_call_item(f"call_{self.calls}", name, json.dumps(args)) + + def message(self, text: str) -> Dict[str, Any]: + self.calls += 1 + return message_item(text, item_id=f"msg_{self.calls}") + + def tool_call_turn(self, name: str, args: Any) -> Dict[str, Any]: + return self.turn(self.call(name, args)) + + def text_turn(self, text: str) -> Dict[str, Any]: + return self.turn(self.message(text)) + + +@pytest.fixture() +def script() -> Iterator[Script]: + yield Script() + + +def _call_id(item: Mapping[str, Any]) -> Any: + return item.get("callId", item.get("call_id")) + + +def assert_well_formed_history(messages: Any) -> None: + """Every function_call in `messages` has a matching output.""" + items = messages if isinstance(messages, list) else [] + call_ids = {_call_id(i) for i in items if isinstance(i, Mapping) and i.get("type") == "function_call"} + output_ids = {_call_id(i) for i in items if isinstance(i, Mapping) and i.get("type") == "function_call_output"} + missing = call_ids - output_ids + assert not missing, f"function_call(s) without a matching output: {sorted(missing)}" + + +def outputs_in(messages: Any) -> List[Mapping[str, Any]]: + return [m for m in messages or [] if isinstance(m, Mapping) and m.get("type") == "function_call_output"] + + +def collect(hooks: HooksManager, hook_name: str = "DoomLoopDetected") -> List[Dict[str, Any]]: + collected: List[Dict[str, Any]] = [] + + def handler(payload: Dict[str, Any], ctx: Any) -> None: + collected.append(payload) + + hooks.on(hook_name, HookEntry(handler=handler)) + return collected + + +def actions_and_streaks(detections: List[Dict[str, Any]]) -> List[List[Any]]: + return [[d["action"], d["streak"]] for d in detections] + + +class Counter: + def __init__(self, result: Any) -> None: + self.count = 0 + self.result = result + + def __call__(self, params: Any, ctx: Any) -> Any: + self.count += 1 + return self.result + + +search_tool = tool( + name="web_search", + description="Search the web.", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: args["query"].strip().lower(), + execute=lambda p, c: {"results": [f"result for {p['query']}"]}, +) + + +def note_tool() -> Dict[str, Any]: + return tool(name="note", input_schema=dict, output_schema=dict, execute=lambda p, c: {"ok": True}) + + +# --------------------------------------------------------------------------- +# B3 -- parallel same-round duplicates +# --------------------------------------------------------------------------- + + +async def test_a_6_wide_identical_fan_out_executes_fully_with_zero_detections(script: Script) -> None: + execute = Counter({"results": ["r"]}) + fanout_tool = tool(name="web_search", input_schema=dict, output_schema=dict, execute=execute) + hooks = HooksManager() + detections = collect(hooks) + args = {"query": "same"} + client = QueuedClient( + [script.turn(*[script.call("web_search", args) for _ in range(6)]), script.text_turn("All six ran.")] + ) + + text = await call_model( + client, {"model": "test-model", "input": "Fan out.", "tools": [fanout_tool], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert text == "All six ran." + # One round = one increment (streak 1): everything executes, no verdicts. + assert execute.count == 6 + assert detections == [] + + +async def test_duplicates_of_a_call_blocked_this_round_share_the_block_decision(script: Script) -> None: + execute = Counter({"ok": True}) + noisy = tool(name="noisy", input_schema=dict, output_schema=dict, execute=execute) + hooks = HooksManager() + detections = collect(hooks) + client = QueuedClient( + [ + script.tool_call_turn("noisy", {}), + script.tool_call_turn("noisy", {}), + script.turn(script.call("noisy", {}), script.call("noisy", {}), script.call("noisy", {})), + script.text_turn("done"), + ] + ) + + text = await call_model( + client, {"model": "test-model", "input": "go", "tools": [noisy], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert text == "done" + # Rounds 1+2 executed one call each; round 3's three duplicates all blocked. + assert execute.count == 2 + assert actions_and_streaks(detections) == [["observe", 2], ["block", 3]] + # Every duplicate received the shared block decision's output. + round_3_outputs = outputs_in(client.requests[3]["input"])[-3:] + assert [o["call_id"] for o in round_3_outputs] == ["call_3", "call_4", "call_5"] + assert all("Doom loop suspected" in o["output"] for o in round_3_outputs) + + +# --------------------------------------------------------------------------- +# B1 + D7 -- state integrity and session reason on text-verdict stops +# --------------------------------------------------------------------------- + + +async def test_text_stop_on_a_response_that_also_carries_tool_calls_leaves_no_dangling_function_call( + script: Script, +) -> None: + accessor = MemoryStateAccessor() + hooks = HooksManager() + session_ends = collect(hooks, "SessionEnd") + doom_text = " ".join(["I am stuck."] * 20) + client = QueuedClient([script.turn(script.message(doom_text), script.call("web_search", {"query": "never runs"}))]) + + await call_model( + client, + { + "model": "test-model", + "input": "go", + "tools": [search_tool], + "doom_loop": True, + "state": accessor, + "hooks": hooks, + }, + ).get_response() + + # Only the initial request was made -- the loop halted before executing. + assert len(client.requests) == 1 + assert len(session_ends) == 1 + assert session_ends[0]["reason"] == "doom_loop" + + state = accessor.stored + assert state is not None + assert_well_formed_history(state.messages) + outputs = outputs_in(state.messages) + assert len(outputs) == 1 + assert "halted by doom-loop detection" in outputs[0]["output"] + + +async def test_a_resumed_conversation_after_a_sealed_text_stop_sends_well_formed_input(script: Script) -> None: + accessor = MemoryStateAccessor() + doom_text = " ".join(["I am stuck."] * 20) + client = QueuedClient( + [ + script.turn(script.message(doom_text), script.call("web_search", {"query": "never runs"})), + script.text_turn("resumed fine"), + ] + ) + await call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "doom_loop": True, "state": accessor} + ).get_response() + + text = await call_model( + client, + { + "model": "test-model", + "input": "continue please", + "tools": [search_tool], + "doom_loop": True, + "state": accessor, + }, + ).get_text() + + assert text == "resumed fine" + assert len(client.requests) == 2 + assert_well_formed_history(client.requests[1]["input"]) + + +async def test_text_only_no_tools_stop_reports_session_end_reason_doom_loop(script: Script) -> None: + hooks = HooksManager() + session_ends = collect(hooks, "SessionEnd") + doom_text = " ".join(["I am stuck."] * 20) + client = QueuedClient([script.text_turn(doom_text)]) + + result = call_model(client, {"model": "test-model", "input": "Summarize.", "doom_loop": True, "hooks": hooks}) + text = await result.get_text() + + assert text == doom_text + assert len(session_ends) == 1 + assert session_ends[0]["reason"] == "doom_loop" + verdict = await result.get_doom_loop_verdict() + assert verdict is not None + assert (verdict["detector"], verdict["action"]) == ("text-repetition", "stop") + + +# --------------------------------------------------------------------------- +# D1 -- allow_final_response gating +# --------------------------------------------------------------------------- + + +async def test_stop_when_plus_doom_stop_during_the_halted_turn_skips_the_final_response_request( + script: Script, +) -> None: + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + # A third scripted turn exists; it must NEVER be requested. + script.text_turn("should never be fetched"), + ] + ) + + result = call_model( + client, + { + "model": "test-model", + "input": "go", + "tools": [search_tool], + "doom_loop": {"ladder": {"observe": 2, "block": 2, "stop": 2}}, + "stop_when": lambda ctx: len(ctx["steps"]) >= 1, + "allow_final_response": True, + }, + ) + await result.get_response() + + # Initial + the tool-round follow-up that stop_when then halts. The final + # text-coercion request is SKIPPED because the doom stop armed while + # executing the halted turn's calls. + assert len(client.requests) == 2 + verdict = await result.get_doom_loop_verdict() + assert verdict is not None + assert (verdict["action"], verdict["streak"]) == ("stop", 2) + + +# --------------------------------------------------------------------------- +# D2 + D3 -- approval resume gating, verdict persistence and clearing +# --------------------------------------------------------------------------- + +approval_tool = tool( + name="risky", + input_schema=dict, + output_schema=dict, + require_approval=True, + execute=lambda p, c: {"done": True}, +) +APPROVAL_LADDER: Dict[str, Any] = {"ladder": {"observe": 1, "block": False, "stop": 2}} + + +async def test_doom_stop_armed_on_the_approval_continuation_makes_no_further_request(script: Script) -> None: + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + script.tool_call_turn("risky", {"target": "same"}), + script.tool_call_turn("risky", {"target": "same"}), + ] + ) + + # Run 1: risky(same) -> pauses for approval. The checkpoint runs BEFORE + # approval partitioning, so the paused call is already recorded (streak 1). + run1 = call_model( + client, + { + "model": "test-model", + "input": "do the risky thing", + "tools": [approval_tool], + "doom_loop": APPROVAL_LADDER, + "tool_choice": "required", + "state": accessor, + }, + ) + await run1.get_response() + paused = accessor.stored + assert paused is not None + assert paused.status == "awaiting_approval" + assert paused.pending_tool_calls is not None + paused_call_id = paused.pending_tool_calls[0].id + assert len(client.requests) == 1 + assert paused.doom_loop is not None + assert paused.doom_loop["tools"]["risky"]["streak"] == 1 + + # Resume: approve -> executes, unsent-results request runs, the model + # calls the same tool again -> the checkpoint crosses the stop rung before + # a second approval pause or PreToolUse can begin. + resume1 = call_model( + client, + { + "model": "test-model", + "tools": [approval_tool], + "doom_loop": APPROVAL_LADDER, + "tool_choice": "required", + "state": accessor, + "approve_tool_calls": [paused_call_id], + }, + ) + requests_before_resume = len(client.requests) + try: + await resume1.get_response() + except Exception: # noqa: BLE001 - upstream `.catch(() => undefined)` + pass + + verdict = await resume1.get_doom_loop_verdict() + assert verdict is not None + assert (verdict["action"], verdict["streak"], verdict["tool_name"]) == ("stop", 2, "risky") + # No additional model request after the doom stop. + assert len(client.requests) == requests_before_resume + 1 + + final_state = accessor.stored + assert final_state is not None + assert final_state.doom_loop is not None + stop_verdict = final_state.doom_loop["stop_verdict"] + assert (stop_verdict["action"], stop_verdict["streak"]) == ("stop", 2) + # Doom stop is terminal: a later fresh run must not inherit the consumed + # forced-choice key from the approval continuation. + assert final_state.consumed_forced_tool_choice_key is None + # The condemned call was never paused for approval again. + assert final_state.status == "complete" + assert_well_formed_history(final_state.messages) + + +async def test_a_doom_blocked_sibling_of_an_approval_pause_is_persisted_as_a_rejected_unsent_result( + script: Script, +) -> None: + """Port-added: pins upstream handleApprovalCheck's `executableNow = [..., + ...blockedCalls]` -- a call refused by the doom checkpoint in a response + that pauses for approval is neither executed nor dropped: it is persisted + as a rejected unsent result carrying the verdict message, so the resume + sends a well-formed history.""" + accessor = MemoryStateAccessor() + execute = Counter({"results": ["r"]}) + counted_search = tool(name="web_search", input_schema=dict, output_schema=dict, execute=execute) + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.turn(script.call("web_search", {"query": "same"}), script.call("risky", {"target": "t"})), + ] + ) + + await call_model( + client, + { + "model": "test-model", + "input": "go", + "tools": [counted_search, approval_tool], + "doom_loop": {"ladder": {"observe": 1, "block": 2, "stop": 6}}, + "state": accessor, + }, + ).get_response() + + state = accessor.stored + assert state is not None + assert state.status == "awaiting_approval" + assert state.pending_tool_calls is not None + assert [c.name for c in state.pending_tool_calls] == ["risky"] + # Round 1 executed; the round-2 repeat was blocked, never executed. + assert execute.count == 1 + assert state.unsent_tool_results is not None + assert [(r.call_id, r.name) for r in state.unsent_tool_results] == [("call_2", "web_search")] + assert "Doom loop suspected" in str(state.unsent_tool_results[0].error) + assert len(client.requests) == 2 + + +async def test_a_decision_only_resume_of_a_condemned_run_halts_without_a_model_request(script: Script) -> None: + """Condemnation rule, decision-resume half (upstream processApprovalDecisions + doom gate): a persisted stop verdict is re-armed on approve/reject resumes, + so the approved call runs but no unsent-results request is made.""" + accessor = MemoryStateAccessor() + hooks = HooksManager() + session_ends = collect(hooks, "SessionEnd") + client = QueuedClient([script.tool_call_turn("risky", {"target": "same"})]) + await call_model( + client, + { + "model": "test-model", + "input": "do it", + "tools": [approval_tool], + "doom_loop": True, + "state": accessor, + }, + ).get_response() + paused = accessor.stored + assert paused is not None + assert paused.pending_tool_calls is not None + paused_call_id = paused.pending_tool_calls[0].id + condemned = { + "detector": "tool-fingerprint", + "action": "stop", + "streak": 6, + "fingerprint": "f" * 64, + "tool_name": "risky", + "message": "condemned", + } + accessor.stored = dataclasses.replace(paused, doom_loop={**(paused.doom_loop or {}), "stop_verdict": condemned}) + + execute_calls = Counter({"done": True}) + counted = tool(name="risky", input_schema=dict, output_schema=dict, require_approval=True, execute=execute_calls) + result = call_model( + client, + { + "model": "test-model", + "tools": [counted], + "doom_loop": True, + "state": accessor, + "approve_tool_calls": [paused_call_id], + "hooks": hooks, + }, + ) + assert await result.get_doom_loop_verdict() == condemned + with pytest.raises(RuntimeError, match="Response not available"): + await result.get_response() + + # The approved call executed exactly once; no model request followed. + assert execute_calls.count == 1 + assert len(client.requests) == 1 + state = accessor.stored + assert state is not None + assert state.status == "in_progress" + assert state.unsent_tool_results is not None + assert [r.call_id for r in state.unsent_tool_results] == [paused_call_id] + assert state.doom_loop is not None + assert state.doom_loop["stop_verdict"] == condemned + assert [e["reason"] for e in session_ends] == ["doom_loop"] + + +async def test_a_fresh_conversational_turn_clears_the_persisted_stop_verdict_streaks_survive( + script: Script, +) -> None: + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "x"}), + script.text_turn("paused here"), + script.text_turn("recovered"), + ] + ) + await call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "doom_loop": True, "state": accessor} + ).get_text() + base = accessor.stored + assert base is not None + # Inject a stop verdict as if the run had been condemned. + accessor.stored = dataclasses.replace( + base, + doom_loop={ + **(base.doom_loop or {"tools": {}}), + "stop_verdict": { + "detector": "tool-fingerprint", + "action": "stop", + "streak": 6, + "fingerprint": "f" * 64, + "tool_name": "web_search", + "message": "condemned", + }, + }, + ) + + # Fresh conversational turn (plain input, no decisions): the verdict clears. + result = call_model( + client, + { + "model": "test-model", + "input": "try something new", + "tools": [search_tool], + "doom_loop": True, + "state": accessor, + }, + ) + assert await result.get_text() == "recovered" + assert await result.get_doom_loop_verdict() is None + state = accessor.stored + assert state is not None + assert state.doom_loop is not None + # Cleared from persistence too (the next save omits it). + assert "stop_verdict" not in state.doom_loop + # Streaks survived the clear. + assert state.doom_loop["tools"]["web_search"]["streak"] == 1 + + +# --------------------------------------------------------------------------- +# D4 -- steer delivery +# --------------------------------------------------------------------------- + +STEER_LADDER: Dict[str, Any] = {"ladder": {"observe": 2, "steer": 2, "block": False, "stop": 6}} + + +def _steer_messages(items: Any) -> List[Mapping[str, Any]]: + return [ + item + for item in items or [] + if isinstance(item, Mapping) + and item.get("role") == "user" + and isinstance(item.get("content"), str) + and "Doom loop suspected" in item["content"] + ] + + +async def test_injects_corrective_guidance_before_the_next_model_request(script: Script) -> None: + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "same"}), + script.tool_call_turn("web_search", {"query": "same"}), + script.text_turn("adjusted course"), + ] + ) + + text = await call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "doom_loop": STEER_LADDER} + ).get_text() + + assert text == "adjusted course" + # The steer verdict fired on round 2; the guidance is a user message in + # the THIRD request's input -- and in no earlier one. + assert len(_steer_messages(client.requests[2]["input"])) == 1 + assert _steer_messages(client.requests[1]["input"]) == [] + + +async def test_steer_queued_at_a_hitl_pause_persists_in_pending_steer_and_delivers_on_resume( + script: Script, +) -> None: + accessor = MemoryStateAccessor() + hitl_tool = tool(name="ask_human", input_schema=dict, output_schema=dict, on_tool_called=lambda p, c: None) + steer_tool = tool(name="lookup", input_schema=dict, output_schema=dict, execute=lambda p, c: {"value": "v"}) + client = QueuedClient( + [ + script.tool_call_turn("lookup", {"key": "same"}), + script.turn(script.call("lookup", {"key": "same"}), script.call("ask_human", {"question": "help?"})), + script.text_turn("thanks"), + ] + ) + + await call_model( + client, + { + "model": "test-model", + "input": "go", + "tools": [steer_tool, hitl_tool], + "doom_loop": STEER_LADDER, + "state": accessor, + }, + ).get_response() + + paused = accessor.stored + assert paused is not None + assert paused.status == "awaiting_hitl" + assert len(client.requests) == 2 + # Not injected mid-pause (invalid history) but persisted for the resume. + assert paused.doom_loop is not None + pending = paused.doom_loop["pending_steer"] + assert len(pending) == 1 + assert "Doom loop suspected" in pending[0] + assert _steer_messages(paused.messages) == [] + + # Upstream stops at the persisted assertion; this port also pins the + # restore half: the queue is rehydrated on resume (and still persisted, + # because the resumed run has no safe flush point before it completes). + assert paused.pending_tool_calls is not None + hitl_call_id = paused.pending_tool_calls[0].id + resumed = call_model( + client, + { + "model": "test-model", + "input": [{"type": "function_call_output", "callId": hitl_call_id, "output": '{"answer": "yes"}'}], + "tools": [steer_tool, hitl_tool], + "doom_loop": STEER_LADDER, + "state": accessor, + }, + ) + assert await resumed.get_text() == "thanks" + assert accessor.stored is not None + assert accessor.stored.doom_loop is not None + assert accessor.stored.doom_loop["pending_steer"] == pending + + +# --------------------------------------------------------------------------- +# D5 + API1 -- loop_key declaration forms +# --------------------------------------------------------------------------- + + +async def test_subset_loop_key_function_same_command_in_a_different_cwd_is_not_a_loop(script: Script) -> None: + runs: List[str] = [] + + def run(params: Any, ctx: Any) -> Dict[str, Any]: + runs.append(f"{params['cwd']}:{params['command']}") + return {"stdout": "ok"} + + bash_tool = tool( + name="bash", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: {"command": args["command"], "cwd": args["cwd"]}, + execute=run, + ) + client = QueuedClient( + [ + script.tool_call_turn("bash", {"command": "ls", "cwd": "/a"}), + script.tool_call_turn("bash", {"command": "ls", "cwd": "/b"}), + script.text_turn("both listed"), + ] + ) + + text = await call_model( + client, {"model": "test-model", "input": "list", "tools": [bash_tool], "doom_loop": {"ladder": {"block": 2}}} + ).get_text() + + assert text == "both listed" + assert runs == ["/a:ls", "/b:ls"] + + +async def test_function_loop_key_collapses_excluded_field_variation(script: Script) -> None: + bash_tool = tool( + name="bash", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: {"command": args["command"], "cwd": args["cwd"]}, + execute=lambda p, c: {"stdout": "ok"}, + ) + hooks = HooksManager() + detections = collect(hooks) + client = QueuedClient( + [ + script.tool_call_turn("bash", {"command": "ls", "cwd": "/a", "verbose": False}), + script.tool_call_turn("bash", {"command": "ls", "cwd": "/a", "verbose": True}), + script.text_turn("done"), + ] + ) + + await call_model( + client, {"model": "test-model", "input": "list", "tools": [bash_tool], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert actions_and_streaks(detections) == [["observe", 2]] + + +async def test_loop_key_false_statically_exempts_a_tool(script: Script) -> None: + poll_tool = tool( + name="check_status", + input_schema=dict, + output_schema=dict, + loop_key=False, + execute=lambda p, c: {"done": False}, + ) + hooks = HooksManager() + detections = collect(hooks) + client = QueuedClient( + [ + *[script.tool_call_turn("check_status", {"jobId": "job-1"}) for _ in range(5)], + script.text_turn("still running"), + ] + ) + + text = await call_model( + client, {"model": "test-model", "input": "wait", "tools": [poll_tool], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert text == "still running" + assert detections == [] + assert len(client.requests) == 6 + + +async def test_d5_loop_key_returning_nothing_never_collides_varying_args_onto_one_identity(script: Script) -> None: + """Upstream: `loopKey: () => undefined` falls back to full args with a + "returned undefined" warning. Python has no `undefined`: a bare `return` + yields `None`, which this port defines as "exempt this call" (upstream's + `null`; doom_loop.py module docstring). The invariant ported is the one + the finding is about -- varying args never collide onto a constant + identity, so no false streak -- not the warning text.""" + buggy_tool = tool( + name="buggy", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: None, + execute=lambda p, c: {"ok": True}, + ) + hooks = HooksManager() + detections = collect(hooks) + client = QueuedClient( + [ + script.tool_call_turn("buggy", {"value": "a"}), + script.tool_call_turn("buggy", {"value": "b"}), + script.tool_call_turn("buggy", {"value": "c"}), + script.text_turn("done"), + ] + ) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + text = await call_model( + client, {"model": "test-model", "input": "go", "tools": [buggy_tool], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert text == "done" + assert detections == [] + # Divergence: exempt, so no fallback warning is emitted. + assert [str(w.message) for w in caught if "DoomLoop" in str(w.message)] == [] + + +# --------------------------------------------------------------------------- +# H1 -- documented miss: nonce evasion +# --------------------------------------------------------------------------- + + +async def test_a_model_inventing_a_fresh_nonce_each_turn_is_not_detected_without_a_loop_key(script: Script) -> None: + nonce_tool = tool( + name="lookup", input_schema=dict, output_schema=dict, execute=lambda p, c: {"value": "same answer every time"} + ) + hooks = HooksManager() + detections = collect(hooks) + client = QueuedClient( + [ + *[script.tool_call_turn("lookup", {"key": "same", "request_id": f"r{i}"}) for i in range(6)], + script.text_turn("gave up"), + ] + ) + + text = await call_model( + client, {"model": "test-model", "input": "go", "tools": [nonce_tool], "doom_loop": True, "hooks": hooks} + ).get_text() + + # The documented limit: locked so the gap is visible, not silent. + assert text == "gave up" + assert detections == [] + + +async def test_the_same_loop_is_caught_when_the_tool_declares_a_loop_key_over_the_meaningful_fields( + script: Script, +) -> None: + nonce_tool = tool( + name="lookup", + input_schema=dict, + output_schema=dict, + loop_key=lambda args: {"key": args["key"]}, + execute=lambda p, c: {"value": "same"}, + ) + hooks = HooksManager() + detections = collect(hooks) + client = QueuedClient( + [ + *[script.tool_call_turn("lookup", {"key": "same", "request_id": f"r{i}"}) for i in range(3)], + script.text_turn("done"), + ] + ) + + await call_model( + client, {"model": "test-model", "input": "go", "tools": [nonce_tool], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert actions_and_streaks(detections) == [["observe", 2], ["block", 3]] + + +# --------------------------------------------------------------------------- +# H2 -- server-tool fingerprinting +# --------------------------------------------------------------------------- + + +def server_search_item(script: Script, query: str) -> Dict[str, Any]: + script.calls += 1 + return { + "type": "web_search_call", + "id": f"ws_{script.calls}", + "status": "completed", + "action": {"type": "search", "query": query}, + } + + +async def test_repeated_identical_server_tool_calls_build_a_streak_and_can_stop_the_run(script: Script) -> None: + hooks = HooksManager() + detections = collect(hooks) + client = QueuedClient( + [ + *[script.turn(server_search_item(script, "same query"), script.call("note", {"n": n})) for n in (1, 2, 3)], + script.text_turn("done"), + ] + ) + + result = call_model( + client, + { + "model": "test-model", + "input": "search", + "tools": [note_tool()], + # block is meaningless for server tools -- downgrades to observe. + "doom_loop": {"ladder": {"observe": 2, "block": 3, "stop": 3}}, + "hooks": hooks, + }, + ) + await result.get_response() + + # Server streak: 1, 2 (observe), 3 (stop) -- exactly 3 requests. + assert len(client.requests) == 3 + assert [d["detector"] for d in detections] == ["server-tool-fingerprint", "server-tool-fingerprint"] + assert [d["action"] for d in detections] == ["observe", "stop"] + verdict = await result.get_doom_loop_verdict() + assert verdict is not None + assert (verdict["detector"], verdict["action"], verdict["streak"], verdict["tool_name"]) == ( + "server-tool-fingerprint", + "stop", + 3, + "server:web_search_call", + ) + + +async def test_distinct_server_tool_queries_do_not_build_a_streak(script: Script) -> None: + hooks = HooksManager() + detections = collect(hooks) + client = QueuedClient( + [ + script.turn(server_search_item(script, "first"), script.call("note", {"n": 1})), + script.turn(server_search_item(script, "second"), script.call("note", {"n": 2})), + script.text_turn("done"), + ] + ) + + text = await call_model( + client, {"model": "test-model", "input": "search", "tools": [note_tool()], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert text == "done" + assert detections == [] + + +async def test_a_proto_field_on_a_server_tool_item_participates_in_the_fingerprint(script: Script) -> None: + """Upstream guards a JS prototype-setter hazard. Python dict keys are inert + data, so the invariant (an own `__proto__` field is part of the identity) + holds structurally; ported to keep it pinned.""" + hooks = HooksManager() + detections = collect(hooks) + + def proto_item(marker: str) -> Dict[str, Any]: + script.calls += 1 + return { + **json.loads(f'{{"__proto__":"{marker}"}}'), + "type": "web_search_call", + "id": f"ws_{script.calls}", + "status": "completed", + "action": {"type": "search", "query": "same query"}, + } + + client = QueuedClient( + [ + *[script.turn(proto_item(marker), script.call("note", {"n": n})) for n, marker in enumerate("abc", 1)], + script.text_turn("done"), + ] + ) + + text = await call_model( + client, {"model": "test-model", "input": "search", "tools": [note_tool()], "doom_loop": True, "hooks": hooks} + ).get_text() + + assert text == "done" + assert detections == [] + + +# --------------------------------------------------------------------------- +# Config lifecycle across resumes +# --------------------------------------------------------------------------- + + +async def test_on_to_off_stale_doom_loop_blob_is_inert_off_run_repeats_freely(script: Script) -> None: + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "x"}), + script.tool_call_turn("web_search", {"query": "x"}), + script.text_turn("run 1 done"), + script.tool_call_turn("web_search", {"query": "x"}), + script.tool_call_turn("web_search", {"query": "x"}), + script.text_turn("run 2 done"), + ] + ) + await call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "doom_loop": True, "state": accessor} + ).get_text() + assert accessor.stored is not None + assert accessor.stored.doom_loop is not None + assert accessor.stored.doom_loop["tools"]["web_search"]["streak"] == 2 + + hooks = HooksManager() + detections = collect(hooks) + text = await call_model( + client, {"model": "test-model", "input": "more", "tools": [search_tool], "state": accessor, "hooks": hooks} + ).get_text() + + assert text == "run 2 done" + assert detections == [] + assert len(client.requests) == 6 + for request_index in (4, 5): + assert "Doom loop" not in outputs_in(client.requests[request_index]["input"])[-1]["output"] + + +async def test_on_to_on_streaks_carry_across_the_resume(script: Script) -> None: + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + script.tool_call_turn("web_search", {"query": "persistent"}), + script.tool_call_turn("web_search", {"query": "persistent"}), + script.text_turn("pausing"), + script.tool_call_turn("web_search", {"query": "persistent"}), + script.text_turn("recovered"), + ] + ) + await call_model( + client, {"model": "test-model", "input": "go", "tools": [search_tool], "doom_loop": True, "state": accessor} + ).get_text() + + hooks = HooksManager() + detections = collect(hooks) + await call_model( + client, + { + "model": "test-model", + "input": "continue", + "tools": [search_tool], + "doom_loop": True, + "state": accessor, + "hooks": hooks, + }, + ).get_text() + + assert actions_and_streaks(detections) == [["block", 3]] diff --git a/tests/unit/test_final_response_validation.py b/tests/unit/test_final_response_validation.py new file mode 100644 index 0000000..6caa3cd --- /dev/null +++ b/tests/unit/test_final_response_validation.py @@ -0,0 +1,176 @@ +"""Port of upstream `tests/unit/final-response-validation.test.ts` (issue #45). + +Pins the specific diagnostics of the final-response validator so its message +cannot silently regress to something ambiguous. The validator is private and +the not-an-array / missing-field shapes are unreachable through the public +`call_model` path, so -- like upstream, which casts to reach +`validateFinalResponse` -- these drive `ModelResult._validate_final_response` +directly. No request is ever dispatched: the validator is pure. +""" + +from __future__ import annotations + +import re +from typing import Any, Callable, Dict + +import pytest + +from openrouter_agent import call_model +from tests._fixtures import QueuedClient, function_call_item, make_response, message_item + +Validator = Callable[..., None] + + +def validator() -> Validator: + client = QueuedClient([]) + result = call_model(client, {"model": "test-model", "input": "hello"}) + validate: Validator = result._validate_final_response + + def checked(*args: Any) -> None: + validate(*args) + # The validator must never touch the transport. + assert client.requests == [] + + return checked + + +MISSING = object() + + +def response(response_id: Any = "resp_1", output: Any = MISSING) -> Dict[str, Any]: + built = make_response("resp_1", []) + if response_id is MISSING: + del built["id"] + else: + built["id"] = response_id + if output is not MISSING: + built["output"] = output + else: + del built["output"] + return built + + +def error_message(fn: Callable[[], None]) -> str: + with pytest.raises(ValueError) as info: + fn() + return str(info.value) + + +# -- empty output array -------------------------------------------------------- + + +async def test_empty_reports_the_empty_array_its_length_and_the_response_id() -> None: + validate = validator() + with pytest.raises(ValueError, match=r'output array is empty \(length 0\) for response "resp_empty"'): + validate(response("resp_empty", [])) + + +async def test_empty_points_at_the_strict_and_allow_final_response_escape_hatches() -> None: + validate = validator() + # Upstream names `strictFinalResponse/allowFinalResponse`; the port names its + # own snake_case options (model_result.py `_validate_final_response`). + with pytest.raises(ValueError, match=r"strict_final_response/allow_final_response"): + validate(response(output=[])) + + +async def test_empty_explains_that_the_provider_returned_no_output_items() -> None: + validate = validator() + with pytest.raises(ValueError, match=r"The model returned no output items"): + validate(response(output=[])) + + +async def test_empty_keeps_the_historical_message_prefix() -> None: + validate = validator() + message = error_message(lambda: validate(response(output=[]))) + assert re.match(r"^Invalid final response: empty or invalid output", message) + assert message.startswith( + 'Invalid final response: empty or invalid output \u2014 output array is empty (length 0) for response "resp_1"' + ) + + +async def test_empty_stays_a_single_line() -> None: + validate = validator() + message = error_message(lambda: validate(response(output=[]))) + assert message != "" + assert "\n" not in message + + +async def test_empty_does_not_throw_when_empty_output_is_explicitly_allowed() -> None: + validate = validator() + validate(response(output=[]), True) + + +# -- output present but not an array --------------------------------------- + + +async def test_non_array_names_the_offending_runtime_type() -> None: + validate = validator() + with pytest.raises(ValueError, match=r"output is not an array \(got string\)"): + validate(response(output="not an array")) + + +async def test_non_array_distinguishes_an_object_from_an_array() -> None: + validate = validator() + with pytest.raises(ValueError, match=r"output is not an array \(got object\)"): + validate(response(output={"type": "message"})) + + +async def test_non_array_does_not_claim_the_array_is_empty() -> None: + validate = validator() + message = error_message(lambda: validate(response(output=42))) + assert re.search(r"output is not an array \(got number\)", message) + assert "output array is empty" not in message + + +async def test_non_array_is_still_tolerated_under_allow_empty_output() -> None: + validate = validator() + # allow_empty_output short-circuits before the message is built, so a + # non-array output is also swallowed on the tolerant path. Pre-existing + # upstream behavior, pinned so a future logic fix is deliberate. + validate(response(output="not an array"), True) + + +# -- missing required fields -------------------------------------------------- + + +async def test_missing_names_id_when_only_the_id_is_missing() -> None: + validate = validator() + with pytest.raises(ValueError, match=r"missing required fields: id$"): + validate(response(MISSING, [{"type": "message"}])) + + +async def test_missing_names_output_when_only_output_is_missing() -> None: + validate = validator() + with pytest.raises(ValueError, match=r"missing required fields: output$"): + validate(response()) + + +async def test_missing_names_both_fields_when_both_are_missing() -> None: + validate = validator() + with pytest.raises(ValueError, match=r"missing required fields: id, output$"): + validate(response(MISSING)) + + +async def test_missing_treats_a_none_response_as_missing_both_fields_rather_than_crashing() -> None: + validate = validator() + message = error_message(lambda: validate(None)) + assert message == "Invalid final response: missing required fields: id, output" + + +async def test_missing_keeps_the_historical_message_prefix() -> None: + validate = validator() + message = error_message(lambda: validate(response())) + assert re.match(r"^Invalid final response: missing required fields", message) + + +# -- valid outputs -------------------------------------------------------------- + + +async def test_valid_accepts_a_tool_call_only_output() -> None: + validate = validator() + validate(response(output=[function_call_item("call_1", "do_thing")])) + + +async def test_valid_accepts_a_normal_message_output() -> None: + validate = validator() + validate(response(output=[message_item("hi")])) diff --git a/tests/unit/test_get_usage.py b/tests/unit/test_get_usage.py new file mode 100644 index 0000000..3845b2a --- /dev/null +++ b/tests/unit/test_get_usage.py @@ -0,0 +1,374 @@ +"""Port of upstream `tests/unit/get-usage.test.ts` -- the pull-based usage +accessor `await result.get_usage()`. + +Python totals are snake_case: `model_calls`, `input_tokens`, `output_tokens`, +`total_tokens`, `cached_tokens`, `reasoning_tokens`, and `cost` (present only +when a response reported it). Usage blocks are snake_case too +(`_extract_model_call_usage` in model_result.py reads `input_tokens`, +`input_tokens_details.cached_tokens`, `output_tokens_details.reasoning_tokens`, +`cost`). +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, cast + +import pytest +from pydantic import BaseModel + +from openrouter_agent import HookEntry, HookName, HooksManager, call_model, tool +from tests._fixtures import ( + MemoryStateAccessor, + QueuedClient, + QueuedResponses, + text_response, + tool_call_response, + usage_block, +) + + +def usage(**overrides: Any) -> Dict[str, Any]: + """Upstream's usageBlock: 100/50/150 tokens, 25 cached, 10 reasoning, cost 0.002.""" + base: Dict[str, Any] = { + "input_tokens_details": {"cached_tokens": 25}, + "output_tokens_details": {"reasoning_tokens": 10}, + } + base.update(overrides) + return usage_block(**base) + + +ROUND_TWO_USAGE = {"input_tokens": 200, "output_tokens": 100, "total_tokens": 300, "cost": 0.003} + + +class EchoInput(BaseModel): + model_config = {"extra": "allow"} + + +echo_tool = tool(name="echo", description="echo", input_schema=EchoInput, execute=lambda params, ctx: {"ok": True}) + + +class RiskyInput(BaseModel): + target: str + + +class RiskyOutput(BaseModel): + done: bool + + +approval_tool = tool( + name="risky", + description="Does something that needs a human to sign off.", + input_schema=RiskyInput, + output_schema=RiskyOutput, + require_approval=True, + execute=lambda params, ctx: {"done": True}, +) + + +def echo_call(response_id: str, **usage_overrides: Any) -> Dict[str, Any]: + return tool_call_response(response_id, "echo", usage=usage(**usage_overrides)) + + +def risky_call(response_id: str) -> Dict[str, Any]: + return tool_call_response(response_id, "risky", arguments=json.dumps({"target": "prod"}), usage=usage()) + + +def text(response_id: str, **usage_overrides: Any) -> Dict[str, Any]: + return text_response(response_id, "hello back", usage=usage(**usage_overrides)) + + +class FailingResponses(QueuedResponses): + """Transport failure on every send (upstream `{ ok: false, error }`).""" + + async def send_async(self, **kwargs: Any) -> Any: + self.requests.append(kwargs) + raise RuntimeError("api down") + + +class FailingClient(QueuedClient): + def __init__(self) -> None: + super().__init__([]) + self.responses = FailingResponses([]) + self.beta = type("Beta", (), {"responses": self.responses})() + + +async def _pause_on_risky(client: QueuedClient, accessor: MemoryStateAccessor) -> Any: + first = call_model( + client, {"model": "test-model", "input": "do the risky thing", "tools": [approval_tool], "state": accessor} + ) + await first.get_pending_tool_calls() + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + return first + + +def _paused_call_id(accessor: MemoryStateAccessor) -> str: + assert accessor.stored is not None + pending = accessor.stored.pending_tool_calls or [] + assert len(pending) == 1 + return pending[0].id + + +async def test_sums_usage_across_every_round_of_a_multi_round_tool_loop() -> None: + client = QueuedClient([echo_call("r1"), text("r2", **ROUND_TWO_USAGE)]) + + result = call_model(client, {"model": "test-model", "input": "hi", "tools": [echo_tool]}) + await result.get_response() + totals = await result.get_usage() + + assert totals == { + "model_calls": 2, + "input_tokens": 300, + "output_tokens": 150, + "total_tokens": 450, + "cached_tokens": 50, + "reasoning_tokens": 20, + "cost": pytest.approx(0.005), + } + assert len(client.requests) == 2 + # The gap this closes: the final response alone reports only round 2. + final_response = await result.get_response() + assert final_response["usage"]["total_tokens"] == 300 + + +async def test_awaits_run_completion_when_called_without_awaiting_get_response_first() -> None: + client = QueuedClient([echo_call("r1"), text("r2")]) + + result = call_model(client, {"model": "test-model", "input": "hi", "tools": [echo_tool]}) + + # No prior await -- get_usage() must gate on completion itself. + totals = await result.get_usage() + assert totals["model_calls"] == 2 + assert totals["total_tokens"] == 300 + assert len(client.requests) == 2 + + +async def test_accumulates_with_no_hooks_configured() -> None: + client = QueuedClient([text("r1")]) + + result = call_model(client, {"model": "test-model", "input": "hi"}) + await result.get_text() + + assert await result.get_usage() == { + "model_calls": 1, + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "cached_tokens": 25, + "reasoning_tokens": 10, + "cost": 0.002, + } + + +async def test_reports_complete_totals_after_consuming_get_items_stream_on_a_tool_loop() -> None: + client = QueuedClient([echo_call("r1"), text("r2", **ROUND_TWO_USAGE)]) + + result = call_model(client, {"model": "test-model", "input": "hi", "tools": [echo_tool]}) + async for _item in result.get_items_stream(): + pass + + assert await result.get_usage() == { + "model_calls": 2, + "input_tokens": 300, + "output_tokens": 150, + "total_tokens": 450, + "cached_tokens": 50, + "reasoning_tokens": 20, + "cost": pytest.approx(0.005), + } + + +async def test_reports_complete_totals_after_consuming_the_no_tools_get_items_stream() -> None: + client = QueuedClient([text("r1")]) + + result = call_model(client, {"model": "test-model", "input": "hi"}) + async for _item in result.get_items_stream(): + pass + + totals = await result.get_usage() + assert totals["model_calls"] == 1 + assert totals["total_tokens"] == 150 + + +async def test_agrees_with_the_session_end_total_usage_hook_payload() -> None: + client = QueuedClient([echo_call("r1"), text("r2")]) + hooks = HooksManager() + ends: List[Dict[str, Any]] = [] + + def on_end(payload: Dict[str, Any], ctx: Any) -> None: + ends.append(payload) + + hooks.on(HookName.SessionEnd.value, HookEntry(handler=on_end)) + + result = call_model(client, {"model": "test-model", "input": "hi", "tools": [echo_tool], "hooks": hooks}) + await result.get_text() + + assert len(ends) == 1 + assert ends[0]["total_usage"]["model_calls"] == 2 + assert await result.get_usage() == ends[0]["total_usage"] + + +async def test_counts_usage_less_responses_in_model_calls_and_omits_cost() -> None: + client = QueuedClient([text_response("r1", "hello back")]) + + result = call_model(client, {"model": "test-model", "input": "hi"}) + await result.get_text() + + totals = await result.get_usage() + assert totals == { + "model_calls": 1, + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "cached_tokens": 0, + "reasoning_tokens": 0, + } + assert "cost" not in totals + + +async def test_returns_zeroed_totals_when_no_model_call_completed_warning_with_the_swallowed_cause() -> None: + client = FailingClient() + + result = call_model(client, {"model": "test-model", "input": "hi"}) + with pytest.raises(RuntimeError, match="api down"): + await result.get_text() + + # Never-raises must not mean silent. Upstream logs + # console.warn('[getUsage] run failed; reporting totals accrued so far:', err); + # the port emits a UserWarning with the snake_case accessor name. + with pytest.warns(UserWarning, match=r"^\[get_usage\] run failed; reporting totals accrued so far: api down$"): + totals = await result.get_usage() + + assert totals["model_calls"] == 0 + assert totals["total_tokens"] == 0 + assert "cost" not in totals + assert len(client.requests) == 1 + + +# -- paused at awaiting_approval --------------------------------------------- + + +async def test_paused_reports_only_the_model_calls_that_completed_before_the_pause() -> None: + accessor = MemoryStateAccessor() + client = QueuedClient([risky_call("r1")]) + + result = await _pause_on_risky(client, accessor) + assert len(client.requests) == 1 + + assert await result.get_usage() == { + "model_calls": 1, + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "cached_tokens": 25, + "reasoning_tokens": 10, + "cost": 0.002, + } + + +async def test_paused_is_idempotent_and_drives_no_further_model_requests() -> None: + accessor = MemoryStateAccessor() + # The second response is queued precisely so an accidental dispatch would + # show up as both a request-count bump and inflated totals. + client = QueuedClient([risky_call("r1"), text("should-not-be-consumed")]) + + result = await _pause_on_risky(client, accessor) + + first = await result.get_usage() + assert first["model_calls"] == 1 + assert await result.get_usage() == first + assert len(client.requests) == 1 + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + + +async def test_scopes_totals_per_model_result_across_an_approval_resume() -> None: + accessor = MemoryStateAccessor() + client = QueuedClient([risky_call("r1"), text("r2", **ROUND_TWO_USAGE)]) + + first = await _pause_on_risky(client, accessor) + paused_call_id = _paused_call_id(accessor) + + resumed = call_model( + client, + {"model": "test-model", "tools": [approval_tool], "state": accessor, "approve_tool_calls": [paused_call_id]}, + ) + await resumed.get_pending_tool_calls() + + assert len(client.requests) == 2 + # Per-ModelResult, not per-conversation: the resume reports only its own + # generation (r2); r1 stays on `first`. + resumed_totals = await resumed.get_usage() + assert resumed_totals["model_calls"] == 1 + assert resumed_totals["input_tokens"] == 200 + assert resumed_totals["output_tokens"] == 100 + assert resumed_totals["total_tokens"] == 300 + assert resumed_totals["cost"] == 0.003 + + first_totals = await first.get_usage() + assert first_totals["model_calls"] == 1 + assert first_totals["input_tokens"] == 100 + assert first_totals["total_tokens"] == 150 + assert first_totals["cost"] == 0.002 + + +@pytest.mark.xfail( + strict=True, + reason=( + "PORT GAP: upstream getUsage() on an approval-resumed run is observation-only " + "(src/lib/model-result.ts:7704-7712, `isResumingFromApproval` guard skips executeToolsIfNeeded), " + "so after a bare getUsage() the resume generation is counted but state stays " + "'in_progress'. Python get_usage() (model_result.py:1487-1499, `await " + "self._ensure_task()`) drives the whole `_run` task, which settles the conversation " + "to 'complete' as a side effect of reading usage." + ), +) +async def test_does_not_advance_an_approval_resumed_run_when_read_before_the_loop() -> None: + accessor = MemoryStateAccessor() + client = QueuedClient([risky_call("r1"), text("r2")]) + + await _pause_on_risky(client, accessor) + paused_call_id = _paused_call_id(accessor) + + resumed = call_model( + client, + {"model": "test-model", "tools": [approval_tool], "state": accessor, "approve_tool_calls": [paused_call_id]}, + ) + + # get_usage() is the FIRST call on the resumed run. + assert (await resumed.get_usage())["model_calls"] == 1 + assert accessor.stored is not None + assert accessor.stored.status == "in_progress" + + +async def test_counts_a_streaming_resume_generation() -> None: + """Upstream 'counts a STREAMING resume generation without advancing the + loop'. The usage half is ported here; the "without advancing" half (status + stays 'in_progress') is the PORT GAP pinned by the xfail above.""" + accessor = MemoryStateAccessor() + resume_response = text("r2_streamed", **ROUND_TWO_USAGE) + + async def resume_stream() -> AsyncIterator[Dict[str, Any]]: + yield {"type": "response.completed", "response": resume_response, "sequence_number": 0} + + # Bespoke transport: the resume dispatch returns a real event stream. + # QueuedResponses returns whatever is queued; the payload is still built + # by the shared builders. + client = QueuedClient([risky_call("r1"), cast(Dict[str, Any], resume_stream())]) + + await _pause_on_risky(client, accessor) + paused_call_id = _paused_call_id(accessor) + + resumed = call_model( + client, + {"model": "test-model", "tools": [approval_tool], "state": accessor, "approve_tool_calls": [paused_call_id]}, + ) + + totals = await resumed.get_usage() + assert totals["model_calls"] == 1 + assert totals["input_tokens"] == 200 + assert totals["output_tokens"] == 100 + assert totals["total_tokens"] == 300 + assert totals["cost"] == 0.003 + assert len(client.requests) == 2 diff --git a/tests/unit/test_hitl_tool.py b/tests/unit/test_hitl_tool.py new file mode 100644 index 0000000..f531989 --- /dev/null +++ b/tests/unit/test_hitl_tool.py @@ -0,0 +1,889 @@ +"""Ports tests/unit/hitl-tool.test.ts. + +Upstream drives the integration half by constructing `ModelResult` directly +against a mocked `betaResponsesSend`; this port drives the same invariants +through `call_model` with the shared `QueuedClient` (contract, Test Parity: +prefer the public API). + +Layering note: upstream's `executeHITLTool(tool, toolCall, turnContext)` builds +the execute context itself. This port's `execute_hitl_tool(tool, call, context)` +takes an already-built execute context and the context is built (and +`tool_call` synthesized) by the `execute_tool` dispatcher +(tool_executor.py `execute_tool`). The upstream `executeHITLTool` cases are +therefore ported through `execute_tool`, which reaches `execute_hitl_tool` +for every HITL tool; the direct-call divergence is pinned by an xfail below. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional + +import pytest +from pydantic import BaseModel, TypeAdapter + +from openrouter_agent import ( + ToolType, + call_model, + is_auto_resolvable_tool, + is_hitl_tool, + is_manual_tool, + tool, +) +from openrouter_agent.tool_executor import apply_on_response_received_hooks, execute_hitl_tool, execute_tool +from openrouter_agent.tool_types import ConversationState, ParsedToolCall +from tests._fixtures import ( + MemoryStateAccessor, + QueuedClient, + function_call_item, + make_response, + message_item, +) + +TURN_CONTEXT: Dict[str, Any] = {"number_of_turns": 1} + + +class AmountInput(BaseModel): + amount: float + + +class XInput(BaseModel): + x: float + + +class IdInput(BaseModel): + id: str + + +class Ok(BaseModel): + ok: bool + + +class Approved(BaseModel): + approved: bool + + +class OkReviewed(BaseModel): + ok: bool + reviewedAt: float + + +class Recorder: + """Records every invocation and returns a fixed value (or raises).""" + + def __init__(self, returns: Any = None, raises: Optional[BaseException] = None) -> None: + self.calls: List[Any] = [] + self.contexts: List[Any] = [] + self._returns = returns + self._raises = raises + + async def __call__(self, value: Any, context: Any = None) -> Any: + self.calls.append(value) + self.contexts.append(context) + if self._raises is not None: + raise self._raises + return self._returns + + +def make_tool_call(name: str, call_id: str, args: Any) -> ParsedToolCall: + return ParsedToolCall(id=call_id, name=name, arguments=args) + + +def output_item(call_id: str, output: Any) -> Dict[str, Any]: + return {"type": "function_call_output", "id": f"output_{call_id}", "callId": call_id, "output": output} + + +def sent_function_call_outputs(request: Dict[str, Any]) -> List[Dict[str, Any]]: + return [item for item in request["input"] if isinstance(item, dict) and item.get("type") == "function_call_output"] + + +# -- tool() factory: HITL tools --------------------------------------------- + + +def test_creates_a_hitl_tool_when_on_tool_called_is_present() -> None: + async def on_tool_called(params: Any, ctx: Any) -> Any: + return {"ok": True} if params.amount < 100 else None + + async def on_response_received(raw: Any, ctx: Any) -> Any: + return raw + + approve = tool( + name="approve_payment", + description="Approve a payment", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=on_tool_called, + on_response_received=on_response_received, + ) + + assert approve["type"] == ToolType.Function.value + assert approve["function"]["name"] == "approve_payment" + assert is_hitl_tool(approve) is True + assert is_manual_tool(approve) is False + assert is_auto_resolvable_tool(approve) is True + assert "execute" not in approve["function"] + + +def test_omits_on_response_received_when_not_provided() -> None: + t = tool(name="approve", input_schema=XInput, output_schema=Ok, on_tool_called=Recorder({"ok": True})) + + assert "on_response_received" not in t["function"] + + +def test_raises_when_a_hitl_tool_is_created_without_an_output_schema() -> None: + with pytest.raises(ValueError, match="output_schema"): + tool(name="approve", input_schema=XInput, on_tool_called=Recorder({"ok": True})) + + +def test_is_manual_tool_true_only_for_tools_with_neither_execute_nor_on_tool_called() -> None: + manual = tool(name="manual", input_schema=XInput, execute=False) + hitl = tool(name="hitl", input_schema=XInput, output_schema=Ok, on_tool_called=Recorder(None)) + regular = tool(name="regular", input_schema=XInput, execute=Recorder({"y": 1})) + + assert [is_manual_tool(t) for t in (manual, hitl, regular)] == [True, False, False] + + +# -- executeHITLTool (via the execute_tool dispatcher; see module docstring) -- + + +async def test_returns_a_tool_execution_result_when_on_tool_called_returns_a_value() -> None: + async def on_tool_called(params: Any, ctx: Any) -> Any: + return {"ok": params.amount < 100} + + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=on_tool_called) + + result = await execute_tool(t, make_tool_call("approve", "c1", {"amount": 5}), TURN_CONTEXT) + + assert result is not None + assert result.get("error") is None + assert result["tool_call_id"] == "c1" + assert result["tool_name"] == "approve" + assert result["result"].model_dump() == {"ok": True} + + +async def test_threads_the_executed_tool_call_into_the_hook_context() -> None: + # Streaming-loop turn context: number_of_turns only, no tool_call. + hook = Recorder(None) + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=hook) + + result = await execute_tool(t, make_tool_call("approve", "call_stream_1", {"amount": 5}), {"number_of_turns": 1}) + + assert result is None + assert len(hook.contexts) == 1 + assert hook.contexts[0]["tool_call"] == { + "type": "function_call", + "id": "call_stream_1", + "callId": "call_stream_1", + "name": "approve", + "arguments": json.dumps({"amount": 5}, separators=(",", ":")), + } + # The synthesized item is wire-shaped: arguments is a JSON string that + # round-trips to the executed call's arguments. + assert json.loads(hook.contexts[0]["tool_call"]["arguments"]) == {"amount": 5} + + +async def test_prefers_a_caller_provided_turn_context_tool_call_over_the_synthesized_one() -> None: + orchestrator_tool_call = { + "type": "function_call", + "id": "call_rich_1", + "callId": "call_rich_1", + "name": "approve", + "arguments": '{"amount":5}', + "status": "completed", + } + hook = Recorder(None) + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=hook) + + await execute_tool( + t, + make_tool_call("approve", "call_rich_1", {"amount": 5}), + {"number_of_turns": 1, "tool_call": orchestrator_tool_call}, + ) + + assert len(hook.contexts) == 1 + assert hook.contexts[0]["tool_call"] is orchestrator_tool_call + + +async def test_does_not_mutate_the_caller_turn_context_when_synthesizing_tool_call() -> None: + hook = Recorder(None) + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=hook) + turn_context: Dict[str, Any] = {"number_of_turns": 1} + + await execute_tool(t, make_tool_call("approve", "c1", {"amount": 5}), turn_context) + + assert turn_context == {"number_of_turns": 1} + + +async def test_direct_execute_hitl_tool_threads_the_executed_tool_call() -> None: + hook = Recorder(None) + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=hook) + + await execute_hitl_tool(t, make_tool_call("approve", "call_direct", {"amount": 5}), {"number_of_turns": 1}) + + assert len(hook.contexts) == 1 + assert hook.contexts[0]["tool_call"]["callId"] == "call_direct" + + +async def test_returns_none_when_on_tool_called_returns_none_pause() -> None: + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=Recorder(None)) + + assert await execute_tool(t, make_tool_call("approve", "c2", {"amount": 5}), TURN_CONTEXT) is None + + +async def test_captures_thrown_errors_into_the_tool_execution_result() -> None: + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=Recorder(raises=RuntimeError("boom")), + ) + + result = await execute_tool(t, make_tool_call("approve", "c3", {"amount": 5}), TURN_CONTEXT) + + assert result is not None + assert isinstance(result["error"], Exception) + assert str(result["error"]) == "boom" + assert result["result"] is None + + +async def test_validates_on_tool_called_return_value_against_output_schema() -> None: + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=Recorder({"ok": "not-a-bool"})) + + result = await execute_tool(t, make_tool_call("approve", "c4", {"amount": 5}), TURN_CONTEXT) + + assert result is not None + assert isinstance(result["error"], Exception) + assert result["result"] is None + + +# -- executeTool dispatcher with HITL tools --------------------------------- + + +async def test_routes_hitl_tools_through_execute_hitl_tool() -> None: + t = tool( + name="approve", input_schema=AmountInput, output_schema=Approved, on_tool_called=Recorder({"approved": True}) + ) + + result = await execute_tool(t, make_tool_call("approve", "c1", {"amount": 5}), TURN_CONTEXT) + + assert result is not None + assert result.get("error") is None + assert result["result"].model_dump() == {"approved": True} + + +async def test_returns_none_for_hitl_pause_through_the_dispatcher() -> None: + hook = Recorder(None) + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=hook) + + result = await execute_tool(t, make_tool_call("approve", "c1", {"amount": 5}), TURN_CONTEXT) + + assert result is None + assert len(hook.calls) == 1 + + +async def test_threads_the_executed_tool_call_into_execute_context_for_regular_tools_too() -> None: + execute = Recorder({"y": 1}) + t = tool(name="regular", input_schema=XInput, execute=execute) + + await execute_tool(t, make_tool_call("regular", "call_reg_1", {"x": 2}), {"number_of_turns": 1}) + + assert len(execute.contexts) == 1 + assert execute.contexts[0]["tool_call"] == { + "type": "function_call", + "id": "call_reg_1", + "callId": "call_reg_1", + "name": "regular", + "arguments": '{"x":2}', + } + + +# -- applyOnResponseReceivedHooks ------------------------------------------- + + +def _call_item(call_id: str, name: str, args: str = "{}") -> Dict[str, Any]: + return function_call_item(call_id, name, args) + + +async def test_transforms_a_function_call_output_item_when_its_tool_has_on_response_received() -> None: + async def on_response_received(raw: Any, ctx: Any) -> Any: + if not isinstance(raw, dict): + return {"ok": False, "reviewedAt": 1234} + return {**raw, "reviewedAt": 1234} + + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=OkReviewed, + on_tool_called=Recorder(None), + on_response_received=on_response_received, + ) + input_items = [_call_item("c1", "approve"), output_item("c1", json.dumps({"ok": True}))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert isinstance(result, list) + assert len(result) == 2 + assert result[0] is input_items[0] + out = result[1] + assert out["type"] == "function_call_output" + assert out["callId"] == "c1" + assert json.loads(out["output"]) == {"ok": True, "reviewedAt": 1234} + # The caller's item is not mutated in place. + assert json.loads(input_items[1]["output"]) == {"ok": True} + + +async def test_does_not_synthesize_a_tool_call_for_on_response_received() -> None: + hook = Recorder({"ok": True}) + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=Recorder(None), + on_response_received=hook, + ) + input_items = [_call_item("c1", "approve"), output_item("c1", json.dumps({"ok": True}))] + + # A streaming-shaped turn context: no caller-provided tool_call. + await apply_on_response_received_hooks(input_items, [t], {"number_of_turns": 1}) + + assert len(hook.calls) == 1 + assert "tool_call" not in hook.contexts[0] + + +async def test_on_response_received_still_sees_a_caller_provided_tool_call() -> None: + # Upstream tool-executor.ts:937-942: no synthetic toolCall, but a + # caller-provided turnContext.toolCall still flows through. + caller_tool_call = {"type": "function_call", "id": "c1", "callId": "c1", "name": "approve", "arguments": "{}"} + hook = Recorder({"ok": True}) + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=Recorder(None), + on_response_received=hook, + ) + input_items = [_call_item("c1", "approve"), output_item("c1", json.dumps({"ok": True}))] + + await apply_on_response_received_hooks(input_items, [t], {"number_of_turns": 1, "tool_call": caller_tool_call}) + + assert len(hook.contexts) == 1 + assert hook.contexts[0]["tool_call"] is caller_tool_call + + +async def test_leaves_output_unchanged_when_no_matching_tool_has_a_hook() -> None: + t = tool(name="regular", input_schema=XInput, execute=Recorder({"y": 1})) + input_items = [_call_item("c1", "regular"), output_item("c1", json.dumps({"y": 1}))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert result is input_items + + +@pytest.mark.xfail( + strict=True, + reason=( + "PORT GAP: upstream applyOnResponseReceivedHooks only considers HITL tools " + "(buildHitlToolMap, tool-executor.ts:1018-1021); non-HITL outputs are never validated or " + "rewritten. Python apply_on_response_received_hooks (tool_executor.py) matches ANY client " + "tool by name and, with no hook, validates the caller output against that tool's " + "output_schema, rewriting a regular tool's non-conforming output into an error wrapper." + ), +) +async def test_leaves_non_hitl_tool_outputs_untouched_even_when_they_miss_the_output_schema() -> None: + t = tool(name="regular", input_schema=XInput, output_schema=Ok, execute=Recorder({"ok": True})) + input_items = [_call_item("c1", "regular"), output_item("c1", json.dumps({"ok": "not-a-bool"}))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert result is input_items + + +async def test_replaces_output_with_an_error_object_preserving_the_original_when_the_hook_throws() -> None: + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=Recorder(None), + on_response_received=Recorder(raises=RuntimeError("invalid result")), + ) + original_payload = {"ok": True, "note": "carry-me-through"} + input_items = [_call_item("c1", "approve"), output_item("c1", json.dumps(original_payload))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + parsed = json.loads(result[1]["output"]) + assert parsed == {"error": "invalid result", "originalOutput": original_payload} + + +async def test_preserves_the_raw_string_as_original_output_when_not_json_and_the_hook_throws() -> None: + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=Recorder(None), + on_response_received=Recorder(raises=RuntimeError("invalid result")), + ) + input_items = [_call_item("c1", "approve"), output_item("c1", "not-json")] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + parsed = json.loads(result[1]["output"]) + assert parsed == {"error": "invalid result", "originalOutput": "not-json"} + + +async def test_passes_the_parsed_raw_result_not_the_raw_string_to_the_hook() -> None: + spy = Recorder({"ok": True, "note": "hi"}) + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=TypeAdapter(Dict[str, Any]), + on_tool_called=Recorder(None), + on_response_received=spy, + ) + payload = {"ok": True, "note": "hi"} + input_items = [_call_item("c1", "approve"), output_item("c1", json.dumps(payload))] + + await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert len(spy.calls) == 1 + assert spy.calls[0] == payload + + +async def test_passes_the_raw_string_through_to_the_hook_when_output_is_not_json() -> None: + spy = Recorder("not-json") + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=TypeAdapter(str), + on_tool_called=Recorder(None), + on_response_received=spy, + ) + input_items = [_call_item("c1", "approve"), output_item("c1", "not-json")] + + await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert spy.calls == ["not-json"] + + +async def test_leaves_outputs_whose_call_id_has_no_matching_function_call_untouched() -> None: + spy = Recorder({"tagged": True}) + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=TypeAdapter(Dict[str, Any]), + on_tool_called=Recorder(None), + on_response_received=spy, + ) + # No function_call in the input at all -- just an orphan output. + input_items = [output_item("orphan", json.dumps({"ok": True}))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert result is input_items + assert spy.calls == [] + + +async def test_validates_raw_output_against_output_schema_without_a_hook_passes() -> None: + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=Recorder(None)) + input_items = [_call_item("c1", "approve"), output_item("c1", json.dumps({"ok": True}))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert result is input_items + + +async def test_replaces_output_with_an_error_object_when_raw_misses_output_schema_and_no_hook() -> None: + t = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=Recorder(None)) + bad_payload = {"ok": "nope"} + input_items = [_call_item("c1", "approve"), output_item("c1", json.dumps(bad_payload))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert result is not input_items + parsed = json.loads(result[1]["output"]) + assert isinstance(parsed["error"], str) + assert len(parsed["error"]) > 0 + assert parsed["originalOutput"] == bad_payload + + +async def test_replaces_output_with_an_error_object_when_the_hook_return_misses_output_schema() -> None: + t = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=Recorder(None), + on_response_received=Recorder({"ok": "not-a-bool"}), + ) + original_payload = {"ok": True} + input_items = [_call_item("c1", "approve"), output_item("c1", json.dumps(original_payload))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + parsed = json.loads(result[1]["output"]) + assert isinstance(parsed["error"], str) + assert len(parsed["error"]) > 0 + assert parsed["originalOutput"] == original_payload + + +async def test_preserves_a_content_array_hook_return_verbatim() -> None: + content_array = [ + {"type": "input_text", "text": "hello"}, + {"type": "input_image", "detail": "auto", "imageUrl": "https://example.com/img.png"}, + ] + t = tool( + name="describe", + input_schema=IdInput, + output_schema=TypeAdapter(List[Dict[str, Any]]), + on_tool_called=Recorder(None), + on_response_received=Recorder(content_array), + ) + input_items = [_call_item("c1", "describe"), output_item("c1", json.dumps({"ok": True}))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + # The output is the content array itself, not a JSON string. + assert isinstance(result[1]["output"], list) + assert result[1]["output"] == content_array + + +async def test_preserves_a_caller_supplied_content_array_output_across_no_hook_validation() -> None: + content_array = [{"type": "input_text", "text": "plain text"}] + t = tool( + name="describe", + input_schema=IdInput, + output_schema=TypeAdapter(List[Dict[str, Any]]), + on_tool_called=Recorder(None), + ) + input_items = [_call_item("c1", "describe"), output_item("c1", content_array)] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert result is input_items + + +@pytest.mark.xfail( + strict=True, + reason=( + "PORT GAP: when a hook fails on a caller-supplied content-array output, upstream " + "formatHookError (tool-executor.ts:849-868) emits a content array: an input_text block " + "carrying JSON {error} followed by the original blocks. Python " + "apply_on_response_received_hooks (tool_executor.py, the `except` branch) always emits " + 'a JSON string {"error", "originalOutput"}, embedding the content array as JSON.' + ), +) +async def test_preserves_a_caller_supplied_content_array_in_original_output_when_the_hook_throws() -> None: + content_array = [{"type": "input_text", "text": "caller payload"}] + t = tool( + name="describe", + input_schema=IdInput, + output_schema=Ok, + on_tool_called=Recorder(None), + on_response_received=Recorder(raises=RuntimeError("bad hook")), + ) + input_items = [_call_item("c1", "describe"), output_item("c1", content_array)] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + out = result[1]["output"] + assert isinstance(out, list) + assert out[0]["type"] == "input_text" + assert json.loads(out[0]["text"]) == {"error": "bad hook"} + assert out[1:] == content_array + + +async def test_ignores_manual_tools_they_are_not_hitl() -> None: + t = tool(name="manual", input_schema=XInput, execute=False) + input_items = [_call_item("c1", "manual"), output_item("c1", json.dumps({"y": 1}))] + + result = await apply_on_response_received_hooks(input_items, [t], TURN_CONTEXT) + + assert result is input_items + + +# -- HITL flow through call_model (integration) ----------------------------- + + +async def test_auto_resolve_on_tool_called_value_feeds_the_model_on_the_next_turn() -> None: + client = QueuedClient( + [ + make_response("resp_turn_0", [function_call_item("c1", "approve", json.dumps({"amount": 5}))]), + make_response("resp_turn_1", [message_item("approved", item_id="msg_1")]), + ] + ) + hook = Recorder({"ok": True}) + approve = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=hook) + + final = await call_model(client, {"model": "test-model", "input": "approve 5", "tools": [approve]}).get_response() + + # Both turns fired: no pause because on_tool_called returned a value. + assert len(client.requests) == 2 + assert len(hook.calls) == 1 + assert final["id"] == "resp_turn_1" + outputs = sent_function_call_outputs(client.requests[1]) + assert len(outputs) == 1 + # Upstream asserts `callId`. This port emits snake_case `call_id`: _send + # normalizes at the transport boundary (model_result.py `_send`). + # Deliberate -- do not change back to callId. + assert outputs[0]["call_id"] == "c1" + assert isinstance(outputs[0]["output"], str) + assert json.loads(outputs[0]["output"]) == {"ok": True} + + +async def test_pause_on_tool_called_returning_none_breaks_the_loop_cleanly() -> None: + client = QueuedClient( + [make_response("resp_turn_0", [function_call_item("c1", "approve", json.dumps({"amount": 5}))])] + ) + approve = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=Recorder(None)) + + result = call_model(client, {"model": "test-model", "input": "approve 5", "tools": [approve]}) + final = await result.get_response() + + # No follow-up request: the caller resumes with its own function_call_output. + assert len(client.requests) == 1 + tool_calls = await result.get_tool_calls() + assert [(c.id, c.name) for c in tool_calls] == [("c1", "approve")] + pending_calls = [item for item in final["output"] if item.get("type") == "function_call"] + assert len(pending_calls) == 1 + assert pending_calls[0]["callId"] == "c1" + + +async def test_resume_caller_output_is_transformed_by_on_response_received_before_the_model_sees_it() -> None: + client = QueuedClient([make_response("resp_resumed", [message_item("approved after review", item_id="msg_1")])]) + on_tool_called = Recorder(None) + + async def on_response_received(raw: Any, ctx: Any) -> Any: + if not isinstance(raw, dict): + return {"ok": False, "reviewedAt": 1234} + return {"ok": raw.get("ok") is True, "reviewedAt": 1234} + + approve = tool( + name="approve", + input_schema=AmountInput, + output_schema=OkReviewed, + on_tool_called=on_tool_called, + on_response_received=on_response_received, + ) + resume_input = [ + function_call_item("c1", "approve", json.dumps({"amount": 5})), + output_item("c1", json.dumps({"ok": True})), + ] + + text = await call_model(client, {"model": "test-model", "input": resume_input, "tools": [approve]}).get_text() + + # Single turn: the caller pre-baked the function_call_output. + assert len(client.requests) == 1 + # on_tool_called does not fire on resume. + assert on_tool_called.calls == [] + outputs = sent_function_call_outputs(client.requests[0]) + assert len(outputs) == 1 + assert isinstance(outputs[0]["output"], str) + assert json.loads(outputs[0]["output"]) == {"ok": True, "reviewedAt": 1234} + assert text == "approved after review" + + +# -- state machine: HITL pause + resume ------------------------------------- + + +async def test_hitl_pause_sets_status_awaiting_hitl_and_populates_pending_tool_calls() -> None: + client = QueuedClient( + [make_response("resp_turn_0", [function_call_item("c1", "approve", json.dumps({"amount": 5}))])] + ) + accessor = MemoryStateAccessor() + approve = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=Recorder(None)) + + result = call_model(client, {"model": "test-model", "input": "approve 5", "tools": [approve], "state": accessor}) + await result.get_response() + + assert len(client.requests) == 1 + saved = accessor.stored + assert saved is not None + assert saved.status == "awaiting_hitl" + assert [c.id for c in saved.pending_tool_calls or []] == ["c1"] + pending = await result.get_pending_tool_calls() + assert [c.id for c in pending] == ["c1"] + assert await result.requires_approval() is True + + +def _awaiting_approval_state() -> ConversationState: + return ConversationState( + id="conv_test", + messages=[function_call_item("c1", "approve", json.dumps({"amount": 5}))], + status="awaiting_approval", + created_at=0, + updated_at=0, + pending_tool_calls=[ParsedToolCall(id="c1", name="approve", arguments={"amount": 5})], + ) + + +async def test_resume_approved_hitl_returning_none_stays_pending_with_status_awaiting_hitl() -> None: + client = QueuedClient([]) + accessor = MemoryStateAccessor() + accessor.stored = _awaiting_approval_state() + on_tool_called = Recorder(None) + approve = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + require_approval=True, + on_tool_called=on_tool_called, + ) + + result = call_model( + client, + { + "model": "test-model", + "input": "approve 5", + "tools": [approve], + "state": accessor, + "approve_tool_calls": ["c1"], + }, + ) + paused_calls = await result.get_pending_tool_calls() + + assert len(on_tool_called.calls) == 1 + # Paused before any follow-up turn. + assert len(client.requests) == 0 + assert [c.id for c in paused_calls] == ["c1"] + saved = accessor.stored + assert saved.status == "awaiting_hitl" + assert [c.id for c in saved.pending_tool_calls or []] == ["c1"] + + +async def test_hitl_only_tool_calls_invoke_on_tool_called_instead_of_being_classified_manual() -> None: + client = QueuedClient( + [ + make_response("resp_turn_0", [function_call_item("c1", "approve", json.dumps({"amount": 5}))]), + make_response("resp_turn_1", [message_item("approved", item_id="msg_1")]), + ] + ) + on_tool_called = Recorder({"ok": True}) + approve = tool(name="approve", input_schema=AmountInput, output_schema=Ok, on_tool_called=on_tool_called) + + final = await call_model(client, {"model": "test-model", "input": "approve 5", "tools": [approve]}).get_response() + + assert len(client.requests) == 2 + assert len(on_tool_called.calls) == 1 + assert final["id"] == "resp_turn_1" + + +async def test_on_response_received_does_not_fire_for_sdk_generated_outputs_on_approval_resume() -> None: + client = QueuedClient([make_response("resp_followup", [message_item("done", item_id="msg_1")])]) + accessor = MemoryStateAccessor() + accessor.stored = _awaiting_approval_state() + on_tool_called = Recorder({"ok": True}) + on_response_received = Recorder({"ok": True, "hooked": True}) + approve = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + require_approval=True, + on_tool_called=on_tool_called, + on_response_received=on_response_received, + ) + + await call_model( + client, + { + "model": "test-model", + "input": "approve 5", + "tools": [approve], + "state": accessor, + "approve_tool_calls": ["c1"], + }, + ).get_response() + + assert len(on_tool_called.calls) == 1 + # SDK-generated outputs are appended verbatim; the hook is reserved for + # caller-supplied outputs. + assert on_response_received.calls == [] + assert len(client.requests) == 1 + outputs = sent_function_call_outputs(client.requests[0]) + assert len(outputs) == 1 + assert isinstance(outputs[0]["output"], str) + assert json.loads(outputs[0]["output"]) == {"ok": True} + + +def _historical_state() -> ConversationState: + return ConversationState( + id="conv_test", + messages=[ + function_call_item("c0", "approve", json.dumps({"amount": 1})), + output_item("c0", json.dumps({"ok": True, "hooked": True})), + ], + status="in_progress", + created_at=0, + updated_at=0, + ) + + +def _fresh_input() -> List[Dict[str, Any]]: + return [ + function_call_item("c1", "approve", json.dumps({"amount": 2})), + output_item("c1", json.dumps({"ok": True})), + ] + + +async def _hook_tagging(raw: Any, ctx: Any) -> Any: + return {**raw, "hooked": True} if isinstance(raw, dict) else {"hooked": True} + + +async def test_on_response_received_fires_only_for_fresh_outputs_not_historical_ones_from_state() -> None: + client = QueuedClient([make_response("resp_reply", [message_item("noted", item_id="msg_1")])]) + accessor = MemoryStateAccessor() + accessor.stored = _historical_state() + hook_calls: List[Any] = [] + + async def on_response_received(raw: Any, ctx: Any) -> Any: + hook_calls.append(raw) + return await _hook_tagging(raw, ctx) + + approve = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=Recorder(None), + on_response_received=on_response_received, + ) + + await call_model( + client, {"model": "test-model", "input": _fresh_input(), "tools": [approve], "state": accessor} + ).get_response() + + # Fired exactly once -- for the fresh c1 output, with the fresh payload. + assert hook_calls == [{"ok": True}] + assert len(client.requests) == 1 + outputs = sent_function_call_outputs(client.requests[0]) + assert [o["call_id"] for o in outputs] == ["c0", "c1"] + # c0 preserved verbatim from history -- NOT re-hooked. + assert json.loads(outputs[0]["output"]) == {"ok": True, "hooked": True} + + +@pytest.mark.xfail( + strict=True, + reason=( + "PORT GAP: upstream invokeOnResponseReceived validates the hook return against outputSchema " + "but serializes the hook's RAW return (tool-executor.ts:944-949 " + "`toFunctionCallOutputValue(hookResult)`), so keys outside the schema survive. Python " + "apply_on_response_received_hooks (tool_executor.py) serializes the VALIDATED value, so a " + "pydantic output_schema strips the extra `hooked` key: c1 is sent as {ok: true}." + ), +) +async def test_fresh_output_carries_the_raw_hook_return_not_the_schema_stripped_value() -> None: + client = QueuedClient([make_response("resp_reply", [message_item("noted", item_id="msg_1")])]) + accessor = MemoryStateAccessor() + accessor.stored = _historical_state() + approve = tool( + name="approve", + input_schema=AmountInput, + output_schema=Ok, + on_tool_called=Recorder(None), + on_response_received=_hook_tagging, + ) + + await call_model( + client, {"model": "test-model", "input": _fresh_input(), "tools": [approve], "state": accessor} + ).get_response() + + outputs = sent_function_call_outputs(client.requests[0]) + c1 = next(o for o in outputs if o["call_id"] == "c1") + assert json.loads(c1["output"]) == {"ok": True, "hooked": True} diff --git a/tests/unit/test_hooks_post_model_call.py b/tests/unit/test_hooks_post_model_call.py new file mode 100644 index 0000000..f0654cd --- /dev/null +++ b/tests/unit/test_hooks_post_model_call.py @@ -0,0 +1,68 @@ +"""Port of upstream `tests/unit/hooks-post-model-call.test.ts`. + +Only the case added by the 680bceb..50a1ec2 sync delta is ported here +("emits when the source close arrives after the terminal event"). The rest of +the upstream file's PostModelCall/SessionEnd invariants are covered in +`tests/unit/test_model_result_hooks.py` (once-per-turn labels, SessionEnd +usage aggregation, SessionEnd on transport failure). +""" + +from __future__ import annotations + +import asyncio +from typing import Any, AsyncIterator, Dict, List, Tuple, cast + + +from openrouter_agent import HookEntry, HookName, HooksManager, call_model +from tests._fixtures import QueuedClient, text_response, usage_block + + +def collect_hooks() -> Tuple[HooksManager, List[Dict[str, Any]], List[Dict[str, Any]]]: + hooks = HooksManager() + calls: List[Dict[str, Any]] = [] + ends: List[Dict[str, Any]] = [] + + def on_call(payload: Dict[str, Any], ctx: Any) -> None: + calls.append(payload) + + def on_end(payload: Dict[str, Any], ctx: Any) -> None: + ends.append(payload) + + hooks.on(HookName.PostModelCall.value, HookEntry(handler=on_call)) + hooks.on(HookName.SessionEnd.value, HookEntry(handler=on_end)) + return hooks, calls, ends + + +async def test_emits_when_the_source_close_arrives_after_the_terminal_event() -> None: + never_closes = asyncio.Event() + streamed = text_response("r_late_close", "hello back", usage=usage_block()) + + async def late_close_stream() -> AsyncIterator[Dict[str, Any]]: + yield {"type": "response.completed", "response": streamed, "sequence_number": 0} + # No close: the source keeps the consumer waiting forever. + await never_closes.wait() + + # Bespoke transport: a streaming body whose payload is still built by the + # shared builders. + client = QueuedClient([cast(Dict[str, Any], late_close_stream())]) + hooks, calls, ends = collect_hooks() + result = call_model(client, {"model": "test-model", "input": "hi", "hooks": hooks}) + + async def drain() -> None: + async for _item in result.get_items_stream(): + pass + + try: + # Not a timing race: the outcome is binary (terminates vs. blocks + # forever). The bound only converts a permanent hang into a failure. + await asyncio.wait_for(drain(), timeout=2.0) + finally: + result.cancel() + + assert len(client.requests) == 1 + assert len(calls) == 1 + assert calls[0]["response_id"] == "r_late_close" + assert calls[0]["turn_type"] == "initial" + assert len(ends) == 1 + assert ends[0]["total_usage"]["model_calls"] == 1 + assert ends[0]["total_usage"]["total_tokens"] == 150 diff --git a/tests/unit/test_hooks_session_lifecycle.py b/tests/unit/test_hooks_session_lifecycle.py new file mode 100644 index 0000000..063c71b --- /dev/null +++ b/tests/unit/test_hooks_session_lifecycle.py @@ -0,0 +1,69 @@ +"""Ports the delta of tests/unit/hooks-session-lifecycle.test.ts. + +The rest of the upstream file (SessionStart/SessionEnd pairing, max_turns, +error-path teardown) is ported in tests/unit/test_model_result_hooks.py; this +file carries the case changed at this sync so the 1:1 mapping exists. + +Upstream constructs `ModelResult` directly (public export) with a request that +still carries client-only fields, on both request-resolution paths: the plain +path and the async-params path. Python `ModelResult._run` resolves every turn's +request through `resolve_async_functions`, which drops client-only keys +(async_params.py `_EXCLUDED`). +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +from openrouter_agent import HookEntry, HookName, HooksManager +from openrouter_agent.model_result import ModelResult +from tests._fixtures import QueuedClient, text_response + + +async def test_strips_hooks_and_active_tools_from_the_outgoing_request_on_both_resolution_paths() -> None: + client = QueuedClient([text_response("resp_1", "hello back"), text_response("resp_2", "hello back")]) + hooks = HooksManager() + events: List[str] = [] + hooks.on(HookName.SessionStart.value, HookEntry(handler=lambda p, c: events.append("start"))) + hooks.on(HookName.SessionEnd.value, HookEntry(handler=lambda p, c: events.append("end"))) + + # Path 1: plain request, constructed directly so client-only fields are + # still present on the request object. Upstream names the field + # `activeTools`; this port's request key is `active_tools`. + direct = ModelResult( + { + "client": client, + "request": {"model": "test-model", "input": "hi", "hooks": hooks, "active_tools": ["echo"]}, + "hooks": hooks, + } + ) + assert await direct.get_text() == "hello back" + + # Path 2: request with an async (callable) param. + async def temperature(turn: Dict[str, Any]) -> float: + return 0.5 + + with_async = ModelResult( + { + "client": client, + "request": { + "model": "test-model", + "input": "hi", + "temperature": temperature, + "hooks": hooks, + "active_tools": ["echo"], + }, + "hooks": hooks, + } + ) + assert await with_async.get_text() == "hello back" + + assert len(client.requests) == 2 + for sent in client.requests: + assert "hooks" not in sent + assert "active_tools" not in sent + assert "activeTools" not in sent + # The async param was resolved, not sent as a callable. + assert client.requests[1]["temperature"] == 0.5 + # The hooks manager still ran for both sessions, in pairs. + assert events == ["start", "end", "start", "end"] diff --git a/tests/unit/test_manual_tool_wire_schema.py b/tests/unit/test_manual_tool_wire_schema.py new file mode 100644 index 0000000..1d5f228 --- /dev/null +++ b/tests/unit/test_manual_tool_wire_schema.py @@ -0,0 +1,168 @@ +"""Ports tests/unit/manual-tool-wire-schema.test.ts. + +`tool(..., wire_input_schema=...)` is a caller-owned JSON Schema sent to the +model verbatim (minus `~`-prefixed metadata keys) in place of the schema +generated from `input_schema`; `input_schema` keeps validating the call. + +The type-level companion (`manual-tool-wire-schema.test-d.ts`) has no runtime +counterpart: Python cannot forbid `wire_input_schema` on executable tools at the +type level (contract, Idiomatic Divergences #4). +""" + +from __future__ import annotations + +import copy +from typing import Any, Dict, List + +from pydantic import BaseModel + +from openrouter_agent import call_model, tool +from openrouter_agent.tool_executor import convert_tools_to_api_format, execute_tool +from openrouter_agent.tool_types import ParsedToolCall +from tests._fixtures import QueuedClient, text_response + + +class ChoiceInput(BaseModel): + choice: str + + +class ValueInput(BaseModel): + value: str + + +class CountInput(BaseModel): + count: float + + +def test_serializes_any_of_and_one_of_while_sanitizing_tilde_prefixed_keys() -> None: + wire_input_schema: Dict[str, Any] = { + "type": "object", + "~rootMetadata": "remove me", + "properties": { + "choice": { + "~propertyMetadata": "remove me", + "anyOf": [{"type": "string"}, {"type": "number"}], + "oneOf": [{"const": "first"}, {"const": "second"}], + }, + }, + "required": ["choice"], + } + manual_tool = tool( + name="choose_value", + description="Choose a value", + input_schema=ChoiceInput, + wire_input_schema=wire_input_schema, + execute=False, + strict=True, + ) + + api = convert_tools_to_api_format([manual_tool]) + + assert len(api) == 1 + emitted = api[0] + assert emitted["type"] == "function" + assert emitted["name"] == "choose_value" + assert emitted["description"] == "Choose a value" + assert emitted["strict"] is True + assert emitted["parameters"] == { + "type": "object", + "properties": { + "choice": { + "anyOf": [{"type": "string"}, {"type": "number"}], + "oneOf": [{"const": "first"}, {"const": "second"}], + }, + }, + "required": ["choice"], + } + + +def test_does_not_mutate_the_caller_owned_schema_and_emits_a_copy() -> None: + wire_input_schema: Dict[str, Any] = { + "type": "object", + "~rootMetadata": True, + "properties": {"value": {"type": "string", "~nestedMetadata": True}}, + } + original_schema = copy.deepcopy(wire_input_schema) + manual_tool = tool( + name="copy_schema", + input_schema=ValueInput, + wire_input_schema=wire_input_schema, + execute=False, + ) + + api = convert_tools_to_api_format([manual_tool]) + parameters = api[0]["parameters"] + + assert wire_input_schema == original_schema + assert parameters is not wire_input_schema + assert parameters["properties"] is not wire_input_schema["properties"] + assert parameters == {"type": "object", "properties": {"value": {"type": "string"}}} + + +def test_falls_back_to_the_model_derived_schema_without_wire_input_schema() -> None: + manual_tool = tool(name="fallback_schema", input_schema=ValueInput, execute=False) + + parameters = convert_tools_to_api_format([manual_tool])[0]["parameters"] + + assert parameters["type"] == "object" + assert parameters["properties"]["value"]["type"] == "string" + assert parameters["required"] == ["value"] + + +def test_uses_the_model_derived_schema_for_executable_shared_context_tools() -> None: + def execute(params: Any, context: Any) -> float: + _ = context["shared"].get("sessionId") + return float(params.count) + + executable_tool = tool(name="shared_context_tool", input_schema=CountInput, execute=execute) + + parameters = convert_tools_to_api_format([executable_tool])[0]["parameters"] + + assert parameters["type"] == "object" + assert parameters["properties"]["count"]["type"] == "number" + + +async def test_input_schema_still_validates_the_call_when_a_wire_schema_is_set() -> None: + # Python-side invariant from the task contract: the wire schema replaces + # only what the model sees; execution still validates against + # `input_schema`. (Upstream's types forbid wireInputSchema on executable + # tools, but the runtime attaches it whenever provided -- tool.ts:687.) + calls: List[Any] = [] + + def execute(params: Any, context: Any) -> str: + calls.append(params) + return params.value + + wired = tool( + name="wired", + input_schema=ValueInput, + wire_input_schema={"type": "object", "properties": {"anything": {}}}, + execute=execute, + ) + + bad = await execute_tool(wired, ParsedToolCall(id="c1", name="wired", arguments={"anything": 1})) + good = await execute_tool(wired, ParsedToolCall(id="c2", name="wired", arguments={"value": "ok"})) + + assert bad is not None and good is not None + assert bad["result"] is None + assert bad["error"] is not None + assert good.get("error") is None + assert good["result"] == "ok" + assert len(calls) == 1 + + +async def test_call_model_sends_the_sanitized_wire_schema_on_the_request() -> None: + client = QueuedClient([text_response("resp_1", "done")]) + manual_tool = tool( + name="choose_value", + input_schema=ChoiceInput, + wire_input_schema={"type": "object", "~meta": 1, "properties": {"choice": {"type": "string"}}}, + execute=False, + ) + + await call_model(client, {"model": "m", "input": "hi", "tools": [manual_tool]}).get_text() + + assert len(client.requests) == 1 + sent_tools = client.requests[0]["tools"] + assert len(sent_tools) == 1 + assert sent_tools[0]["parameters"] == {"type": "object", "properties": {"choice": {"type": "string"}}} diff --git a/tests/unit/test_max_output_tokens_truncation.py b/tests/unit/test_max_output_tokens_truncation.py new file mode 100644 index 0000000..6672f04 --- /dev/null +++ b/tests/unit/test_max_output_tokens_truncation.py @@ -0,0 +1,129 @@ +"""Port of upstream tests/unit/max-output-tokens-truncation.test.ts. + +A turn the provider stopped at `max_output_tokens` carries a cut-off +`function_call` batch. None of it executes, and the loop does not re-request +on the same exhausted budget. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel + +from openrouter_agent import call_model, step_count_is, tool +from openrouter_agent.stream_transformers import extract_tool_calls_from_response, response_has_tool_calls +from tests._fixtures import QueuedClient, function_call_item, make_response + + +def truncated_tool_call_response(output: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: + if output is None: + cut = function_call_item("call_1", "run_shell", '{"commands":') + cut["status"] = "incomplete" + output = [cut] + return make_response( + "resp_truncated", + output, + status="incomplete", + incompleteDetails={"reason": "max_output_tokens"}, + usage={"input_tokens": 4529, "output_tokens": 3002, "total_tokens": 7531}, + ) + + +def truncated_batch_response() -> Dict[str, Any]: + items = [ + function_call_item(f"call_weather_{index}", "get_weather", json.dumps({"city": city})) + for index, city in enumerate(["Paris", "London", "Tokyo"]) + ] + cut = function_call_item("call_shell", "run_shell", '{"commands":') + cut["status"] = "incomplete" + return truncated_tool_call_response([*items, cut]) + + +class ShellInput(BaseModel): + commands: List[str] + + +class WeatherInput(BaseModel): + city: str + + +def test_extracts_no_tool_calls_from_a_response_truncated_at_max_output_tokens() -> None: + response = truncated_tool_call_response() + assert response_has_tool_calls(response) is False + assert extract_tool_calls_from_response(response) == [] + + +def test_other_incomplete_reasons_still_yield_tool_calls() -> None: + response = make_response( + "resp_filtered", + [function_call_item("call_1", "run_shell", '{"commands":["ls"]}')], + status="incomplete", + incompleteDetails={"reason": "content_filter"}, + ) + assert response_has_tool_calls(response) is True + assert [call.id for call in extract_tool_calls_from_response(response)] == ["call_1"] + + +async def test_finalizes_on_the_truncated_turn_without_executing_or_requesting_again() -> None: + executed: List[Any] = [] + + def run_shell(params: ShellInput, ctx: Any) -> Dict[str, Any]: + executed.append(params) + return {"ok": True} + + client = QueuedClient([truncated_tool_call_response()]) + result = call_model( + client, + { + "model": "test-model", + "input": "Run echo hello.", + "stop_when": step_count_is(3), + "tools": [tool(name="run_shell", input_schema=ShellInput, execute=run_shell)], + }, + ) + response = await result.get_response() + + assert executed == [] + assert len(client.requests) == 1 + assert response["id"] == "resp_truncated" + assert response["status"] == "incomplete" + + +async def test_does_not_execute_calls_that_completed_before_the_cut_off_either() -> None: + executed: List[Any] = [] + + def get_weather(params: WeatherInput, ctx: Any) -> Dict[str, Any]: + executed.append(params) + return {"temperature": 22} + + def run_shell(params: ShellInput, ctx: Any) -> Dict[str, Any]: + executed.append(params) + return {"ok": True} + + client = QueuedClient([truncated_batch_response()]) + result = call_model( + client, + { + "model": "test-model", + "input": "Weather in three cities, then run echo hello.", + "stop_when": step_count_is(3), + "tools": [ + tool(name="get_weather", input_schema=WeatherInput, execute=get_weather), + tool(name="run_shell", input_schema=ShellInput, execute=run_shell), + ], + }, + ) + response = await result.get_response() + + assert executed == [] + assert len(client.requests) == 1 + assert response["status"] == "incomplete" + # The caller gets the whole turn, cut-off item included, to resume from. + assert [item.get("callId", item["type"]) for item in response["output"]] == [ + "call_weather_0", + "call_weather_1", + "call_weather_2", + "call_shell", + ] diff --git a/tests/unit/test_mcp_tool_branding.py b/tests/unit/test_mcp_tool_branding.py index 1e6110a..e552c27 100644 --- a/tests/unit/test_mcp_tool_branding.py +++ b/tests/unit/test_mcp_tool_branding.py @@ -58,3 +58,32 @@ async def test_regular_client_tool_result_carries_source_client() -> None: await result.get_response() assert events[0]["source"] == "client" + + +async def test_source_is_discriminated_by_the_mcp_brand_not_the_tool_name() -> None: + # Ports the runtime half of the mcp-result-discrimination.test-d.ts note + # changed at this sync: `source` (driven by the MCP brand) stays the + # discriminant even though names are now preserved, so a client tool + # named like an MCP tool must not be classified as MCP -- and an MCP tool + # sharing a client-ish name must still be. + lookalike = tool(name="mcp_remote_echo", input_schema=dict, output_schema=dict, execute=lambda p, c: {"ok": True}) + branded = mark_mcp(tool(name="local_echo", input_schema=dict, output_schema=dict, execute=lambda p, c: {"ok": 1})) + client = QueuedClient( + [ + make_response( + "r1", + [ + function_call_item("call_a", "mcp_remote_echo", "{}"), + function_call_item("call_b", "local_echo", "{}"), + ], + ), + text_response("r2", "done"), + ] + ) + + result = call_model(client, {"model": "test-model", "input": "hi", "tools": [lookalike, branded]}) + + events = [event async for event in result.get_tool_stream() if event.get("type") == "tool_result"] + + assert [(e["toolCallId"], e["source"]) for e in events] == [("call_a", "client"), ("call_b", "mcp")] + assert len(client.requests) == 2 diff --git a/tests/unit/test_model_result_hooks.py b/tests/unit/test_model_result_hooks.py index a640bb8..25a4681 100644 --- a/tests/unit/test_model_result_hooks.py +++ b/tests/unit/test_model_result_hooks.py @@ -51,7 +51,7 @@ async def test_session_end_reason_max_turns_when_stop_condition_halts() -> None: "model": "test-model", "input": "hi", "tools": [echo_tool], - "stop_when": step_count_is(1), + "stop_when": step_count_is(0), "allow_final_response": False, "hooks": hooks, }, @@ -172,7 +172,7 @@ def stop_handler(payload: Any, ctx: Any) -> Any: "model": "test-model", "input": "hi", "tools": [echo_tool], - "stop_when": step_count_is(1), + "stop_when": step_count_is(0), "allow_final_response": False, "hooks": hooks, }, diff --git a/tests/unit/test_next_turn_params.py b/tests/unit/test_next_turn_params.py new file mode 100644 index 0000000..8a6d97c --- /dev/null +++ b/tests/unit/test_next_turn_params.py @@ -0,0 +1,368 @@ +"""Port of upstream `tests/unit/next-turn-params.test.ts` AND +`src/lib/next-turn-params.test.ts` (both map here). + +Python next-turn param keys are snake_case (`tool_choice`, `max_output_tokens`, +`top_p`, `top_k`, ...) and a tool declares them via +`tool(next_turn_params={"key": fn(args, ctx)})`. +""" + +from __future__ import annotations + +import warnings +from typing import Any, Dict, List, Optional + +import pytest +from pydantic import BaseModel + +from openrouter_agent import tool +from openrouter_agent.next_turn_params import ( + apply_next_turn_params_to_request, + build_next_turn_params_context, + execute_next_turn_params_functions, +) +from openrouter_agent.tool_types import ParsedToolCall + + +class EmptyInput(BaseModel): + pass + + +class QInput(BaseModel): + q: str + + +class DepthInput(BaseModel): + depth: int + + +def create_request(**overrides: Any) -> Dict[str, Any]: + request: Dict[str, Any] = {"model": "openai/gpt-4", "input": [{"role": "user", "content": "hello"}]} + request.update(overrides) + return request + + +def create_tool_call(name: str, args: Any) -> ParsedToolCall: + return ParsedToolCall(id=f"call_{name}", name=name, arguments=args) + + +async def _done(params: Any, ctx: Any) -> str: + return "done" + + +# --------------------------------------------------------------------------- +# build_next_turn_params_context (tests/unit/next-turn-params.test.ts) +# --------------------------------------------------------------------------- + + +def test_build_context_extracts_all_fields_from_a_fully_populated_request() -> None: + request = create_request( + models=["openai/gpt-4", "anthropic/claude"], + temperature=0.7, + max_output_tokens=500, + top_p=0.9, + top_k=40, + instructions="Be terse", + ) + + context = build_next_turn_params_context(request) + + assert context == { + "input": request["input"], + "model": "openai/gpt-4", + "models": ["openai/gpt-4", "anthropic/claude"], + "temperature": 0.7, + "max_output_tokens": 500, + "top_p": 0.9, + "top_k": 40, + "instructions": "Be terse", + } + + +def test_build_context_applies_defaults_for_missing_fields() -> None: + context = build_next_turn_params_context({}) + + # Upstream `topK: undefined` and `toolChoice: undefined` are absent keys here. + assert context == { + "input": [], + "model": "", + "models": [], + "temperature": None, + "max_output_tokens": None, + "top_p": None, + "instructions": None, + } + + +# --------------------------------------------------------------------------- +# execute_next_turn_params_functions (tests/unit/next-turn-params.test.ts) +# --------------------------------------------------------------------------- + + +async def test_execute_returns_an_empty_dict_when_no_tools_have_next_turn_params() -> None: + plain_tool = tool(name="plain", input_schema=QInput, execute=_done) + + result = await execute_next_turn_params_functions( + [create_tool_call("plain", {"q": "x"})], [plain_tool], create_request() + ) + + assert result == {} + + +async def test_execute_computes_parameters_from_tool_call_arguments_and_context() -> None: + search_tool = tool( + name="search", + input_schema=DepthInput, + execute=_done, + next_turn_params={ + "temperature": lambda params, ctx: 0.1 if params["depth"] > 3 else 0.9, + "instructions": lambda params, ctx: f"{ctx.get('instructions') or ''} depth={params['depth']}", + }, + ) + + result = await execute_next_turn_params_functions( + [create_tool_call("search", {"depth": 5})], [search_tool], create_request(instructions="Base.") + ) + + assert result == {"temperature": 0.1, "instructions": "Base. depth=5"} + + +async def test_execute_composes_later_tools_see_modifications_made_by_earlier_tools() -> None: + first_tool = tool( + name="first", input_schema=EmptyInput, execute=_done, next_turn_params={"temperature": lambda p, c: 0.5} + ) + second_tool = tool( + name="second", + input_schema=EmptyInput, + execute=_done, + next_turn_params={"temperature": lambda p, c: (c.get("temperature") or 0) + 0.3}, + ) + + result = await execute_next_turn_params_functions( + [create_tool_call("first", {}), create_tool_call("second", {})], + [first_tool, second_tool], + create_request(temperature=0.7), + ) + + # first sets 0.5; second sees 0.5 (not the original 0.7) and adds 0.3 + assert result["temperature"] == pytest.approx(0.8) + + +async def test_execute_awaits_async_next_turn_params_functions() -> None: + async def pick_model(params: Any, ctx: Any) -> str: + return "openai/gpt-5" + + async_tool = tool(name="async_tool", input_schema=EmptyInput, execute=_done, next_turn_params={"model": pick_model}) + + result = await execute_next_turn_params_functions( + [create_tool_call("async_tool", {})], [async_tool], create_request() + ) + + assert result == {"model": "openai/gpt-5"} + + +async def test_execute_runs_a_tool_once_per_call_when_it_was_called_multiple_times() -> None: + invocations: List[int] = [] + + def count(params: Any, ctx: Any) -> int: + invocations.append(1) + return len(invocations) + + counting_tool = tool( + name="counter", input_schema=EmptyInput, execute=_done, next_turn_params={"max_output_tokens": count} + ) + + result = await execute_next_turn_params_functions( + [create_tool_call("counter", {}), create_tool_call("counter", {})], [counting_tool], create_request() + ) + + assert len(invocations) == 2 + assert result["max_output_tokens"] == 2 + + +async def test_execute_ignores_tools_that_were_not_called() -> None: + called_tool = tool( + name="called", input_schema=EmptyInput, execute=_done, next_turn_params={"temperature": lambda p, c: 0.1} + ) + uncalled_tool = tool( + name="uncalled", input_schema=EmptyInput, execute=_done, next_turn_params={"temperature": lambda p, c: 0.9} + ) + + result = await execute_next_turn_params_functions( + [create_tool_call("called", {})], [called_tool, uncalled_tool], create_request() + ) + + assert result == {"temperature": 0.1} + + +async def test_execute_skips_server_tools_entirely() -> None: + invoked: List[int] = [] + + def server_fn(params: Any, ctx: Any) -> float: + invoked.append(1) + return 0.0 + + server_tool_like = { + "_brand": "server-tool", + "function": {"name": "server_search", "next_turn_params": {"temperature": server_fn}}, + } + + result = await execute_next_turn_params_functions( + [create_tool_call("server_search", {})], [server_tool_like], create_request(temperature=0.7) + ) + + assert result == {} + assert invoked == [] + + +async def test_execute_throws_a_descriptive_error_when_tool_call_arguments_are_not_an_object() -> None: + strict_tool = tool( + name="strict", input_schema=EmptyInput, execute=_done, next_turn_params={"temperature": lambda p, c: 0.5} + ) + + with pytest.raises(Exception, match="Tool call arguments for strict must be an object, got array"): + await execute_next_turn_params_functions( + [create_tool_call("strict", ["not", "an", "object"])], [strict_tool], create_request() + ) + with pytest.raises(Exception, match="Tool call arguments for strict must be an object, got string"): + await execute_next_turn_params_functions( + [create_tool_call("strict", "a-string")], [strict_tool], create_request() + ) + + +async def test_execute_warns_and_skips_invalid_next_turn_params_keys() -> None: + odd_tool = tool( + name="odd", + input_schema=EmptyInput, + execute=_done, + next_turn_params={"bogus_key": lambda p, c: 42, "temperature": lambda p, c: 0.3}, + ) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = await execute_next_turn_params_functions([create_tool_call("odd", {})], [odd_tool], create_request()) + + assert result == {"temperature": 0.3} + messages = [str(w.message) for w in caught] + assert sum('Invalid next_turn_params key "bogus_key" in tool "odd"' in m for m in messages) == 1 + + +async def test_execute_ignores_non_function_values_in_next_turn_params() -> None: + weird_tool = tool(name="weird", input_schema=EmptyInput, execute=_done, next_turn_params={"temperature": 0.5}) + + result = await execute_next_turn_params_functions([create_tool_call("weird", {})], [weird_tool], create_request()) + + assert result == {} + + +# --------------------------------------------------------------------------- +# apply_next_turn_params_to_request (src/lib/next-turn-params.test.ts) +# --------------------------------------------------------------------------- + + +def test_apply_passes_through_non_null_values_unchanged() -> None: + result = apply_next_turn_params_to_request( + create_request(), {"temperature": 0.5, "max_output_tokens": 1000, "instructions": "Be helpful"} + ) + + assert result["temperature"] == 0.5 + assert result["max_output_tokens"] == 1000 + assert result["instructions"] == "Be helpful" + + +def test_apply_converts_null_values_to_absent() -> None: + result = apply_next_turn_params_to_request( + create_request(temperature=0.7), {"temperature": None, "max_output_tokens": None, "instructions": None} + ) + + # Upstream maps null -> undefined (spread overrides with undefined). The + # Python equivalent of an undefined property is an absent key. + assert "temperature" not in result + assert "max_output_tokens" not in result + assert "instructions" not in result + assert result["model"] == "openai/gpt-4" + + +def test_apply_preserves_original_request_fields_not_in_computed_params() -> None: + result = apply_next_turn_params_to_request( + create_request(model="anthropic/claude-3", temperature=0.9), {"max_output_tokens": 500} + ) + + assert result["model"] == "anthropic/claude-3" + assert result["temperature"] == 0.9 + assert result["max_output_tokens"] == 500 + + +def test_apply_handles_empty_computed_params_without_changing_the_request() -> None: + request = create_request(temperature=0.5) + + result = apply_next_turn_params_to_request(request, {}) + + assert result == request + assert result["temperature"] == 0.5 + assert result["model"] == "openai/gpt-4" + + +def test_apply_handles_mixed_null_and_non_null_values() -> None: + result = apply_next_turn_params_to_request( + create_request(temperature=0.7, top_p=0.9), + {"temperature": None, "top_p": 0.5, "max_output_tokens": None, "instructions": "Updated instructions"}, + ) + + assert "temperature" not in result + assert result["top_p"] == 0.5 + assert "max_output_tokens" not in result + assert result["instructions"] == "Updated instructions" + + +def test_apply_preserves_zero_as_a_valid_non_null_value() -> None: + result = apply_next_turn_params_to_request(create_request(), {"temperature": 0, "max_output_tokens": 0}) + + assert result["temperature"] == 0 + assert result["max_output_tokens"] == 0 + + +def test_apply_preserves_empty_string_as_a_valid_non_null_value() -> None: + instructions: Optional[str] = "" + result = apply_next_turn_params_to_request(create_request(), {"instructions": instructions}) + + assert result["instructions"] == "" + + +# tool_choice is how a tool-search tool widens the model's reach mid-run: the +# `tools` array stays identical (preserving the provider's prompt-cache prefix) +# while `{"type": "allowed_tools", "tools": [...]}` grows. + + +def allowed(names: List[str]) -> Dict[str, Any]: + return {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": n} for n in names]} + + +def test_apply_replaces_tool_choice_with_a_widened_allowed_tools_set() -> None: + result = apply_next_turn_params_to_request( + create_request(tool_choice=allowed(["tool_search"])), + {"tool_choice": allowed(["tool_search", "get_weather"])}, + ) + + assert result["tool_choice"] == allowed(["tool_search", "get_weather"]) + + +def test_apply_leaves_the_tools_array_untouched_so_the_prompt_cache_prefix_survives() -> None: + tools = [ + {"type": "function", "name": "tool_search", "parameters": {}}, + {"type": "function", "name": "get_weather", "parameters": {}}, + ] + request = create_request(tools=tools, tool_choice=allowed(["tool_search"])) + + result = apply_next_turn_params_to_request(request, {"tool_choice": allowed(["tool_search", "get_weather"])}) + + # Upstream `toBe(tools)`: same object identity. + assert result["tools"] is tools + + +def test_apply_leaves_tool_choice_alone_when_no_tool_computed_one() -> None: + result = apply_next_turn_params_to_request( + create_request(tool_choice=allowed(["tool_search"])), {"temperature": 0.2} + ) + + assert result["tool_choice"] == allowed(["tool_search"]) + assert result["temperature"] == 0.2 diff --git a/tests/unit/test_next_turn_tool_choice_loop.py b/tests/unit/test_next_turn_tool_choice_loop.py new file mode 100644 index 0000000..2031acc --- /dev/null +++ b/tests/unit/test_next_turn_tool_choice_loop.py @@ -0,0 +1,110 @@ +"""Port of upstream `tests/unit/next-turn-tool-choice-loop.test.ts`. + +`next_turn_params["tool_choice"]` driven through the real `call_model` loop, +asserted against the DISPATCHED request rather than the helper in isolation: +the follow-up builder re-derives the wire tool choice from the caller- +configured value after next-turn params apply, so a hook that merges +correctly can still be discarded before dispatch. Only a loop-level assertion +catches that. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +from pydantic import BaseModel + +from openrouter_agent import call_model, tool +from tests._fixtures import QueuedClient, text_response, tool_call_response + + +class SearchInput(BaseModel): + pattern: str + + +class EmptyInput(BaseModel): + pass + + +def allowed(names: List[str]) -> Dict[str, Any]: + return {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": n} for n in names]} + + +def search_call_response() -> Dict[str, Any]: + return tool_call_response("resp_tool", "tool_search", call_id="call_1", arguments='{"pattern":"weather"}') + + +def make_search_tool(widened: List[str]) -> Dict[str, Any]: + return tool( + name="tool_search", + input_schema=SearchInput, + execute=lambda params, ctx: {"found": widened}, + next_turn_params={"tool_choice": lambda args, ctx: allowed(widened)}, + ) + + +get_weather = tool(name="get_weather", input_schema=EmptyInput, execute=lambda params, ctx: {"ok": True}) + + +def dispatched_tool_choices(client: QueuedClient) -> List[Any]: + return [request.get("tool_choice") for request in client.requests] + + +async def test_reaches_the_dispatched_follow_up_request() -> None: + client = QueuedClient([search_call_response(), text_response("resp_text", "done")]) + + await call_model( + client, + { + "model": "test-model", + "input": "hi", + "tools": [make_search_tool(["tool_search", "get_weather"]), get_weather], + "tool_choice": allowed(["tool_search"]), + }, + ).get_text() + + choices = dispatched_tool_choices(client) + assert len(choices) == 2 + # Turn 1 uses the caller's narrow set. + assert choices[0] == allowed(["tool_search"]) + # Turn 2 carries the tool's widened set, not the caller's original. + assert choices[1] == allowed(["tool_search", "get_weather"]) + + +async def test_leaves_the_tools_array_untouched_while_the_choice_widens() -> None: + client = QueuedClient([search_call_response(), text_response("resp_text", "done")]) + + await call_model( + client, + { + "model": "test-model", + "input": "hi", + "tools": [make_search_tool(["tool_search", "get_weather"]), get_weather], + "tool_choice": allowed(["tool_search"]), + }, + ).get_text() + + assert len(client.requests) == 2 + tool_names = [[t.get("name") for t in (request.get("tools") or [])] for request in client.requests] + # Identical across turns: widening must not change the request prefix, or + # the provider's prompt cache is lost. + assert tool_names[0] == ["tool_search", "get_weather"] + assert tool_names[1] == tool_names[0] + assert client.requests[1]["tools"] == client.requests[0]["tools"] + + +async def test_does_not_disturb_tool_choice_when_no_tool_computes_one() -> None: + plain = tool(name="tool_search", input_schema=SearchInput, execute=lambda params, ctx: {"ok": True}) + client = QueuedClient([search_call_response(), text_response("resp_text", "done")]) + + await call_model( + client, + { + "model": "test-model", + "input": "hi", + "tools": [plain, get_weather], + "tool_choice": allowed(["tool_search"]), + }, + ).get_text() + + assert dispatched_tool_choices(client) == [allowed(["tool_search"]), allowed(["tool_search"])] diff --git a/tests/unit/test_openrouter.py b/tests/unit/test_openrouter.py new file mode 100644 index 0000000..c8a46b3 --- /dev/null +++ b/tests/unit/test_openrouter.py @@ -0,0 +1,192 @@ +"""Ports tests/unit/openrouter.test.ts. + +Upstream inspects `openrouter._options` (the TS SDK's constructor options). The +generated Python SDK exposes the equivalent state on `sdk_configuration` +(`user_agent`, `server_url`, and the registered `SDKHooks` under `_hooks`); +this port additionally keeps the normalized hooks on `agent_hooks` +(openrouter.py). `OpenRouterCore` maps to the generated `openrouter.OpenRouter`. + +Delta at this sync: the three user-agent cases, and the hook assertions moving +from `toHaveLength(1)` to `toContain(hook)` (the SDK may pre-register its own +hooks, so membership of the caller's hook -- not slot length -- is the +invariant). +""" + +from __future__ import annotations + +from typing import Any, Dict + +import httpx +import pytest +from openrouter import OpenRouter as SDKOpenRouter +from openrouter._hooks import SDKHooks + +from openrouter_agent import ModelResult, OpenRouter +from openrouter_agent.user_agent import AGENT_USER_AGENT + +REQUEST: Dict[str, Any] = {"model": "openai/gpt-4o", "input": "Hello"} + + +class NoopBeforeRequest: + def before_request(self, ctx: Any, request: Any) -> Any: + return request + + +class NoopAfterSuccess: + def after_success(self, ctx: Any, response: Any) -> Any: + return response + + +class Combined(NoopBeforeRequest, NoopAfterSuccess): + pass + + +def _client(**kwargs: Any) -> OpenRouter: + """Construct the client with a caller-supplied async httpx client. + + The generated SDK's GC finalizer (openrouter/httpclient.py `close_clients`) + calls `asyncio.run(async_client.aclose())` for an SDK-owned async client + when collected outside a running loop. That swaps the event loop set on the + policy at an arbitrary GC point and trips pytest-asyncio's unclosed-loop + check in whichever async test runs next. A supplied client is never + closed by that finalizer, so these sync tests cannot leak a loop. + """ + kwargs.setdefault("async_client", httpx.AsyncClient()) + return OpenRouter(**kwargs) + + +def _sdk_hooks(client: OpenRouter) -> SDKHooks: + hooks = client.sdk_configuration.__dict__["_hooks"] + assert isinstance(hooks, SDKHooks) + return hooks + + +def test_instantiates_without_options() -> None: + assert isinstance(_client(), OpenRouter) + + +def test_instantiates_with_api_key() -> None: + assert isinstance(_client(api_key="test-key"), OpenRouter) + + +def test_returns_a_model_result_from_call_model() -> None: + client = _client(api_key="test-key") + + assert isinstance(client.call_model(REQUEST), ModelResult) + + +def test_preserves_binding_when_call_model_is_detached() -> None: + client = _client(api_key="test-key") + call_model = client.call_model + + result = call_model(REQUEST) + + assert isinstance(result, ModelResult) + assert result.options["client"] is client + + +def test_accepts_sdk_options_like_server_url() -> None: + assert isinstance(_client(api_key="test-key", server_url="https://custom.endpoint.com"), OpenRouter) + + +def test_defaults_to_the_agent_sdk_user_agent() -> None: + assert _client(api_key="test-key").sdk_configuration.user_agent == AGENT_USER_AGENT + + +def test_preserves_an_explicitly_configured_user_agent() -> None: + client = _client(api_key="test-key", user_agent="custom/1.0") + + assert client.sdk_configuration.user_agent == "custom/1.0" + + +async def test_sends_the_agent_sdk_user_agent_on_outgoing_requests() -> None: + captured: Dict[str, Any] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured.setdefault("user_agents", []).append(request.headers.get("user-agent")) + raise RuntimeError("captured request") + + async_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + client = OpenRouter(api_key="test-key", async_client=async_client) + try: + with pytest.raises(RuntimeError, match="captured request"): + await client.call_model(REQUEST).get_text() + finally: + await async_client.aclose() + + assert captured["user_agents"] == [AGENT_USER_AGENT] + + +def test_is_an_instance_of_the_generated_sdk_client() -> None: + assert isinstance(_client(api_key="test-key"), SDKOpenRouter) + + +def test_passes_constructor_options_through_to_the_instance() -> None: + client = _client(api_key="test-key", server_url="https://custom.endpoint.com") + + assert isinstance(client, SDKOpenRouter) + assert client.sdk_configuration.server_url == "https://custom.endpoint.com" + + +def test_is_usable_as_both_agent_and_sdk_client() -> None: + client = _client(api_key="test-key") + + assert isinstance(client, OpenRouter) + assert isinstance(client, SDKOpenRouter) + assert isinstance(client.call_model(REQUEST), ModelResult) + + +# -- hooks ------------------------------------------------------------------- + + +def test_exposes_an_sdk_hooks_instance_by_default() -> None: + client = _client(api_key="test-key") + + assert isinstance(_sdk_hooks(client), SDKHooks) + # No agent-level hooks were supplied. + assert client.agent_hooks is None + + +def test_uses_a_passed_sdk_hooks_instance_as_is() -> None: + hook = NoopBeforeRequest() + hooks = SDKHooks() + hooks.register_before_request_hook(hook) + + client = _client(api_key="test-key", hooks=hooks) + + assert client.agent_hooks is hooks + assert _sdk_hooks(client) is hooks + assert hook in _sdk_hooks(client).before_request_hooks + + +def test_wraps_a_single_hook_object_into_an_sdk_hooks_instance() -> None: + hook = NoopBeforeRequest() + + client = _client(api_key="test-key", user_agent="custom/1.0", hooks=hook) + + assert isinstance(_sdk_hooks(client), SDKHooks) + assert hook in _sdk_hooks(client).before_request_hooks + assert client.sdk_configuration.user_agent == "custom/1.0" + + +def test_wraps_a_list_of_hook_objects_into_an_sdk_hooks_instance() -> None: + before = NoopBeforeRequest() + after = NoopAfterSuccess() + + client = _client(api_key="test-key", hooks=[before, after]) + + hooks = _sdk_hooks(client) + assert before in hooks.before_request_hooks + assert after in hooks.after_success_hooks + assert after not in hooks.before_request_hooks + assert before not in hooks.after_success_hooks + + +def test_registers_a_multi_interface_hook_under_every_matching_slot() -> None: + combined = Combined() + + client = _client(api_key="test-key", hooks=combined) + + hooks = _sdk_hooks(client) + assert hooks.before_request_hooks.count(combined) == 1 + assert hooks.after_success_hooks.count(combined) == 1 diff --git a/tests/unit/test_replay_buffer_compaction.py b/tests/unit/test_replay_buffer_compaction.py new file mode 100644 index 0000000..afb7dbb --- /dev/null +++ b/tests/unit/test_replay_buffer_compaction.py @@ -0,0 +1,125 @@ +"""Ports `packages/agent/tests/unit/replay-buffer-compaction.test.ts`. + +Replay policy of `ReusableReadableStream`: full replay by default, watermark +start for active-consumers mode, repeated compaction, and terminal-value +handling (``is_terminal_value`` stops the pump and closes the source once). +""" + +from __future__ import annotations + +from typing import AsyncIterator, List + +from openrouter_agent.reusable_stream import BUFFER_COMPACTION_MIN_HEAD, ReusableReadableStream + + +async def source(values: List[int]) -> AsyncIterator[int]: + for value in values: + yield value + + +class EndlessSource: + """Mirrors upstream's `pull()` source: produces 1, 2, 1, 2, ... forever. + + ``aclose`` is the Python analogue of ReadableStream ``cancel()``. + """ + + def __init__(self, *, fail_close: bool = False) -> None: + self.close_count = 0 + self.produced: List[int] = [] + self._fail_close = fail_close + + def __aiter__(self) -> "EndlessSource": + return self + + async def __anext__(self) -> int: + value = 1 if len(self.produced) % 2 == 0 else 2 + self.produced.append(value) + return value + + async def aclose(self) -> None: + self.close_count += 1 + if self._fail_close: + raise RuntimeError("cleanup failed") + + +async def _collect(iterator: AsyncIterator[int]) -> List[int]: + return [value async for value in iterator] + + +async def test_replays_complete_history_to_sequential_and_post_completion_consumers_by_default() -> None: + stream = ReusableReadableStream(source([1, 2, 3])) + + assert await _collect(stream.create_consumer()) == [1, 2, 3] + assert await _collect(stream.create_consumer()) == [1, 2, 3] + + +async def test_starts_new_active_consumer_consumers_at_the_current_watermark() -> None: + stream = ReusableReadableStream(source([1, 2, 3]), stream_replay="active-consumers") + first = stream.create_consumer() + + assert await first.__anext__() == 1 + second = stream.create_consumer() + assert await _collect(first) == [2, 3] + assert await _collect(second) == [2, 3] + + +async def test_continues_to_replay_active_consumer_events_through_repeated_compaction() -> None: + values = list(range(2500)) + # Large enough to cross the compaction threshold more than once. + assert len(values) > 2 * BUFFER_COMPACTION_MIN_HEAD + stream = ReusableReadableStream(source(values), stream_replay="active-consumers") + consumer = stream.create_consumer() + + assert await _collect(consumer) == values + late_consumer = stream.create_consumer() + assert await _collect(late_consumer) == [] + + +async def test_stops_at_a_terminal_value_and_closes_the_source_once() -> None: + # Upstream additionally counts `releaseLock()` calls; Python async + # iterators have no reader lock, so only the close count is ported. + endless = EndlessSource() + stream = ReusableReadableStream(endless, is_terminal_value=lambda value: value == 1) + + assert await _collect(stream.create_consumer()) == [1] + assert endless.close_count == 1 + # The pump stopped at the terminal value instead of reading ahead. + assert endless.produced == [1] + assert stream.is_complete is True + + +async def test_retains_the_terminal_value_when_source_close_fails() -> None: + endless = EndlessSource(fail_close=True) + stream = ReusableReadableStream(endless, is_terminal_value=lambda value: value == 1) + + assert await _collect(stream.create_consumer()) == [1] + assert endless.close_count == 1 + + +async def test_treats_a_failed_terminal_value_as_the_final_buffered_event() -> None: + stream = ReusableReadableStream(source([1, 2]), is_terminal_value=lambda value: value == 2) + + assert await _collect(stream.create_consumer()) == [1, 2] + + +async def test_treats_an_incomplete_terminal_value_as_the_final_buffered_event() -> None: + # Upstream's two terminal cases (failed / incomplete response) share the + # same shape at this layer; both are kept for 1:1 parity. + stream = ReusableReadableStream(source([1, 2, 3]), is_terminal_value=lambda value: value == 2) + + assert await _collect(stream.create_consumer()) == [1, 2] + + +async def test_on_value_observes_every_buffered_value_once_in_order() -> None: + seen: List[int] = [] + stream = ReusableReadableStream( + source([1, 2, 3]), + stream_replay="active-consumers", + on_value=seen.append, + is_terminal_value=lambda value: value == 2, + ) + + assert await _collect(stream.create_consumer()) == [1, 2] + assert seen == [1, 2] + # Captured via on_value even though active-consumers already trimmed it. + assert stream.find_last_buffered(lambda _value: True) is None diff --git a/tests/unit/test_reusable_stream.py b/tests/unit/test_reusable_stream.py new file mode 100644 index 0000000..d4a14c3 --- /dev/null +++ b/tests/unit/test_reusable_stream.py @@ -0,0 +1,257 @@ +"""Ports `packages/agent/tests/unit/reusable-stream.test.ts`. + +Contract tests for the replay-buffer memory/lifecycle edges: + +- active-consumers mode must not pin O(stream) backlog once every consumer has + departed +- full-replay mode (default) keeps replaying everything +- a pending ``__anext__`` must never hang after ``aclose()`` / ``athrow()`` / + ``cancel()`` + +Determinism: a pending read is parked with :func:`_park`, which yields to the +event loop until the reading task is blocked on its waiter. That is scheduler +ordering, not wall-clock timing. Upstream's "would hang" failure mode is caught +by ``asyncio.wait_for`` around the awaited result. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, AsyncIterator, Generic, List, Optional, TypeVar + +import pytest + +from openrouter_agent.reusable_stream import ReusableReadableStream + +T = TypeVar("T") + +# Guard against a regression that re-introduces a hang; not a timing input. +_HANG_GUARD_SECONDS = 5 + + +async def stream_of(values: List[T]) -> AsyncIterator[T]: + for value in values: + yield value + + +class ControlledSource(Generic[T]): + """Source whose producer is driven by the test, for mid-stream assertions.""" + + _CLOSE: Any = object() + + def __init__(self) -> None: + self._queue: asyncio.Queue[Any] = asyncio.Queue() + + def push(self, value: T) -> None: + self._queue.put_nowait(value) + + def close(self) -> None: + self._queue.put_nowait(self._CLOSE) + + def __aiter__(self) -> "ControlledSource[T]": + return self + + async def __anext__(self) -> T: + value = await self._queue.get() + if value is self._CLOSE: + raise StopAsyncIteration + return value # type: ignore[no-any-return] + + +async def _park(task: "asyncio.Task[Any]") -> None: + """Yield until ``task`` is suspended (blocked on its waiter) or finished.""" + for _ in range(10): + await asyncio.sleep(0) + assert not task.done(), "pending read resolved before it could be parked" + + +async def _collect(iterator: AsyncIterator[T]) -> List[T]: + return [value async for value in iterator] + + +async def test_retains_full_replay_for_late_consumers_by_default() -> None: + stream = ReusableReadableStream(stream_of([1, 2])) + first = stream.create_consumer() + assert await _collect(first) == [1, 2] + + second = stream.create_consumer() + assert await _collect(second) == [1, 2] + + +async def test_active_consumers_backlog_follows_slowest_remaining_consumer() -> None: + source: ControlledSource[int] = ControlledSource() + stream = ReusableReadableStream(source, stream_replay="active-consumers") + fast = stream.create_consumer() + slow = stream.create_consumer() + + source.push(1) + source.push(2) + assert await fast.__anext__() == 1 + assert await fast.__anext__() == 2 + await fast.aclose() + + # The slow consumer still reads from position 0 — trimming follows the + # slowest attached consumer, never the fastest. + assert await slow.__anext__() == 1 + assert await slow.__anext__() == 2 + source.close() + with pytest.raises(StopAsyncIteration): + await slow.__anext__() + + +async def test_active_consumers_backlog_dropped_once_every_consumer_departs() -> None: + source: ControlledSource[int] = ControlledSource() + stream = ReusableReadableStream(source, stream_replay="active-consumers") + only = stream.create_consumer() + source.push(1) + source.push(2) + assert await only.__anext__() == 1 + assert await only.__anext__() == 2 + await only.aclose() + + # Nobody is attached anymore: the retained backlog is dropped. + assert stream.find_last_buffered(lambda _value: True) is None + + # A late consumer starts at the watermark instead of replaying history. + source.push(3) + late = stream.create_consumer() + assert await late.__anext__() == 3 + source.close() + assert await _collect(late) == [] + + +async def test_active_consumers_fully_caught_up_consumer_leaves_nothing_for_later_joiners() -> None: + # Trimming follows consumption, not attachment time. Replay-after-detach is + # the documented trade-off of active-consumers mode; "full" keeps it. + stream = ReusableReadableStream(stream_of([1, 2]), stream_replay="active-consumers") + only = stream.create_consumer() + assert await _collect(only) == [1, 2] + + late = stream.create_consumer() + assert await _collect(late) == [] + + +async def test_aclose_wakes_a_pending_next_instead_of_hanging() -> None: + source: ControlledSource[int] = ControlledSource() + stream = ReusableReadableStream(source) + consumer = stream.create_consumer() + source.push(1) + assert await consumer.__anext__() == 1 + + pending = asyncio.ensure_future(consumer.__anext__()) + await _park(pending) + await consumer.aclose() + with pytest.raises(StopAsyncIteration): + await asyncio.wait_for(pending, _HANG_GUARD_SECONDS) + source.close() + + +async def test_athrow_rejects_a_pending_next_with_the_same_error() -> None: + source: ControlledSource[int] = ControlledSource() + stream = ReusableReadableStream(source) + consumer = stream.create_consumer() + source.push(1) + assert await consumer.__anext__() == 1 + + pending = asyncio.ensure_future(consumer.__anext__()) + await _park(pending) + boom = RuntimeError("boom") + with pytest.raises(RuntimeError, match="boom") as thrown: + await consumer.athrow(boom) + assert thrown.value is boom + with pytest.raises(RuntimeError, match="boom"): + await asyncio.wait_for(pending, _HANG_GUARD_SECONDS) + source.close() + + +async def test_athrow_normalizes_an_exception_type() -> None: + source: ControlledSource[int] = ControlledSource() + stream = ReusableReadableStream(source) + consumer = stream.create_consumer() + with pytest.raises(ValueError): + await consumer.athrow(ValueError) + # The consumer is detached afterwards. + with pytest.raises(StopAsyncIteration): + await consumer.__anext__() + source.close() + + +async def test_cancel_settles_waiters_and_leaves_no_buffered_backlog() -> None: + source: ControlledSource[int] = ControlledSource() + stream = ReusableReadableStream(source) + consumer = stream.create_consumer() + source.push(1) + source.push(2) + + # Whether the parked read wakes with a buffered value or as done depends on + # interleaving with the pump; the contract is only that it settles and + # that no backlog survives. + pending = asyncio.ensure_future(consumer.__anext__()) + results = await asyncio.wait_for( + asyncio.gather(pending, stream.cancel(), return_exceptions=True), + _HANG_GUARD_SECONDS, + ) + assert results[1] is None + assert results[0] == 1 or isinstance(results[0], StopAsyncIteration) + + assert stream.find_last_buffered(lambda _value: True) is None + assert stream.is_complete is True + fresh = stream.create_consumer() + assert await _collect(fresh) == [] + + +async def test_cancel_closes_the_source_exactly_once() -> None: + closed: List[str] = [] + + class Source: + def __aiter__(self) -> "Source": + return self + + async def __anext__(self) -> int: + await asyncio.Event().wait() # never produces; cancel() must unblock + raise AssertionError("unreachable") + + async def aclose(self) -> None: + closed.append("closed") + + stream = ReusableReadableStream(Source()) + consumer = stream.create_consumer() + pending = asyncio.ensure_future(consumer.__anext__()) + await _park(pending) + + await stream.cancel() + await stream.cancel() + + with pytest.raises(StopAsyncIteration): + await asyncio.wait_for(pending, _HANG_GUARD_SECONDS) + assert closed == ["closed"] + + +async def test_source_error_propagates_after_buffered_values() -> None: + async def failing() -> AsyncIterator[int]: + yield 1 + raise RuntimeError("source failed") + + stream = ReusableReadableStream(failing()) + consumer = stream.create_consumer() + received: List[int] = [] + with pytest.raises(RuntimeError, match="source failed"): + async for value in consumer: + received.append(value) + assert received == [1] + + +async def test_find_last_buffered_scans_from_the_end() -> None: + stream = ReusableReadableStream(stream_of([{"type": "a", "n": 1}, {"type": "b", "n": 2}, {"type": "a", "n": 3}])) + assert len(await _collect(stream.create_consumer())) == 3 + + found: Optional[dict] = stream.find_last_buffered(lambda item: item["type"] == "a") + assert found == {"type": "a", "n": 3} + assert stream.find_last_buffered(lambda item: item["type"] == "missing") is None + + +async def test_none_is_a_valid_stream_value() -> None: + # The port uses a private sentinel for cleared slots (upstream uses + # `undefined`), so `None` must survive replay and compaction. + stream = ReusableReadableStream(stream_of([None, 1, None]), stream_replay="active-consumers") + assert await _collect(stream.create_consumer()) == [None, 1, None] diff --git a/tests/unit/test_server_tool.py b/tests/unit/test_server_tool.py new file mode 100644 index 0000000..8b07f2d --- /dev/null +++ b/tests/unit/test_server_tool.py @@ -0,0 +1,171 @@ +"""Ports tests/unit/server-tool.test.ts. + +Type-level cases (`expectTypeOf`, `@ts-expect-error` config narrowing, guard +narrowing, the `ToolResultItem` union) have no Python runtime counterpart +(contract, Idiomatic Divergences #4); their runtime halves are ported below. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from pydantic import BaseModel + +from openrouter_agent import is_client_tool, is_server_tool, server_tool, tool +from openrouter_agent.tool_executor import convert_tools_to_api_format + + +class EchoInput(BaseModel): + msg: str + + +def _echo(params: Any, context: Any) -> str: + return params.msg + + +# -- server_tool() ------------------------------------------------------------- + + +def test_creates_a_branded_server_tool_carrying_the_sdk_config_through() -> None: + t = server_tool({"type": "web_search_2025_08_26", "engine": "exa", "maxResults": 10}) + + assert t["_brand"] == "server-tool" + assert t["config"] == {"type": "web_search_2025_08_26", "engine": "exa", "maxResults": 10} + assert t["id"] == "server:web_search_2025_08_26" + assert is_server_tool(t) is True + assert is_client_tool(t) is False + + +def test_allows_overriding_the_stable_tool_set_id() -> None: + t = server_tool({"type": "web_search_2025_08_26"}, {"id": "server:public_search"}) + + assert t["id"] == "server:public_search" + + +def test_rejects_an_empty_stable_tool_set_id() -> None: + with pytest.raises(ValueError, match="must not be empty"): + server_tool({"type": "openrouter:datetime"}, {"id": ""}) + + +def test_carries_type_specific_config_fields_through() -> None: + # Runtime half of upstream "narrows config shape based on the chosen type + # literal"; the narrowing itself is TS type-level only. + dt = server_tool({"type": "openrouter:datetime", "parameters": {"timezone": "America/New_York"}}) + img = server_tool({"type": "image_generation", "size": "1024x1024", "quality": "high"}) + + assert dt["config"]["parameters"]["timezone"] == "America/New_York" + assert img["config"]["size"] == "1024x1024" + assert img["config"]["quality"] == "high" + + +def test_server_tool_copies_the_caller_config() -> None: + config = {"type": "openrouter:datetime"} + t = server_tool(config) + config["type"] = "mutated" + + assert t["config"] == {"type": "openrouter:datetime"} + + +# -- convert_tools_to_api_format --------------------------------------------- + + +def test_passes_server_tool_configs_through_verbatim() -> None: + tools = [ + server_tool({"type": "openrouter:datetime"}), + server_tool({"type": "web_search_2025_08_26", "engine": "native"}), + ] + + api = convert_tools_to_api_format(tools) + + # The stable tool-set id and brand are client-only and never reach the wire. + assert api == [ + {"type": "openrouter:datetime"}, + {"type": "web_search_2025_08_26", "engine": "native"}, + ] + + +def test_shapes_client_tools_into_the_function_call_api_format() -> None: + client_tool = tool(name="echo", input_schema=EchoInput, execute=_echo) + + api = convert_tools_to_api_format([client_tool]) + + assert len(api) == 1 + assert api[0]["type"] == "function" + assert api[0]["name"] == "echo" + assert api[0]["parameters"]["properties"]["msg"]["type"] == "string" + # DIVERGENCE: upstream sends `description: null` for an undeclared + # description (tool-executor.ts:130 `description || null`). This port omits + # the key instead (tool_executor.py convert_tools_to_api_format). Both mean + # "no description" to the Responses API. + assert "description" not in api[0] + + +def test_serializes_strict_true_from_the_tool_definition() -> None: + strict_tool = tool(name="echo", input_schema=EchoInput, strict=True, execute=_echo) + + api = convert_tools_to_api_format([strict_tool]) + + assert api[0]["type"] == "function" + assert api[0]["name"] == "echo" + assert api[0]["strict"] is True + + +@pytest.mark.skip(reason="Out of scope: unified `run`/lifecycle tools are not ported (async/deferred tools).") +def test_serializes_strict_true_from_a_unified_run_tool() -> None: + raise AssertionError("not ported") + + +@pytest.mark.skip(reason="Out of scope: `tool.agent` (agent tools) is not ported.") +def test_serializes_strict_true_from_an_agent_tool() -> None: + raise AssertionError("not ported") + + +def test_serializes_strict_false_as_false_instead_of_null() -> None: + non_strict_tool = tool(name="echo", input_schema=EchoInput, strict=False, execute=_echo) + + api = convert_tools_to_api_format([non_strict_tool]) + + assert api[0]["type"] == "function" + assert "strict" in api[0] + assert api[0]["strict"] is False + + +def test_omits_strict_when_not_declared() -> None: + plain_tool = tool(name="echo", input_schema=EchoInput, execute=_echo) + + api = convert_tools_to_api_format([plain_tool]) + + assert api[0]["type"] == "function" + # DIVERGENCE: upstream "defaults strict to null when not declared" and + # sends `strict: null` (tool-executor.ts:131 `strict ?? null`). This port + # omits the key when `strict` is unset (tool_executor.py + # convert_tools_to_api_format: `if fn.get("strict") is not None`). An + # absent key and an explicit null are equivalent on the wire; do not + # "fix" this to assert `api[0]["strict"] is None`. + assert "strict" not in api[0] + + +def test_mixes_client_and_server_tools_in_one_array() -> None: + tools = [ + tool(name="echo", input_schema=EchoInput, execute=_echo), + server_tool({"type": "image_generation", "size": "1024x1024"}), + ] + + api = convert_tools_to_api_format(tools) + + assert len(api) == 2 + assert api[0]["type"] == "function" + assert api[0]["name"] == "echo" + assert api[1] == {"type": "image_generation", "size": "1024x1024"} + + +# -- guards (runtime half of the type-level narrowing test) ------------------ + + +def test_guards_partition_a_mixed_tool_list() -> None: + client = tool(name="x", input_schema=EchoInput, execute=lambda params, ctx: 1) + server = server_tool({"type": "openrouter:datetime"}) + + assert [is_server_tool(t) for t in (client, server)] == [False, True] + assert [is_client_tool(t) for t in (client, server)] == [True, False] diff --git a/tests/unit/test_state_context_and_broadcaster.py b/tests/unit/test_state_context_and_broadcaster.py index ba049b6..2da6d24 100644 --- a/tests/unit/test_state_context_and_broadcaster.py +++ b/tests/unit/test_state_context_and_broadcaster.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +from typing import Any from openrouter_agent import ( ToolContextStore, @@ -46,7 +47,7 @@ def test_tool_context_local_and_shared_mutations_are_live() -> None: async def test_tool_event_broadcaster_replays_to_multiple_consumers() -> None: - broadcaster = ToolEventBroadcaster() + broadcaster: ToolEventBroadcaster[Any] = ToolEventBroadcaster() first = broadcaster.create_consumer() broadcaster.push({"n": 1}) broadcaster.push({"n": 2}) @@ -58,7 +59,7 @@ async def test_tool_event_broadcaster_replays_to_multiple_consumers() -> None: async def test_tool_event_broadcaster_waits_for_future_events() -> None: - broadcaster = ToolEventBroadcaster() + broadcaster: ToolEventBroadcaster[Any] = ToolEventBroadcaster() async def consume(): return [item async for item in broadcaster.create_consumer()] diff --git a/tests/unit/test_stop_conditions.py b/tests/unit/test_stop_conditions.py new file mode 100644 index 0000000..525ca29 --- /dev/null +++ b/tests/unit/test_stop_conditions.py @@ -0,0 +1,174 @@ +"""Ports `packages/agent/tests/unit/stop-conditions.test.ts`.""" + +from __future__ import annotations + +from typing import Any, List, Optional + +import pytest + +from openrouter_agent import finish_reason_is, has_tool_call, max_cost, max_tokens_used, step_count_is +from openrouter_agent.stop_conditions import is_stop_condition_met +from openrouter_agent.tool_types import ParsedToolCall, StepResult + +_UNSET: Any = object() + + +def make_step( + *, + tool_call_names: Optional[List[str]] = None, + usage: Any = _UNSET, + finish_reason: Optional[str] = None, +) -> StepResult: + """A minimal StepResult; stop conditions only read tool_calls, usage and + finish_reason, so the remaining fields are inert placeholders.""" + return StepResult( + step_type="initial", + text="", + tool_calls=[ParsedToolCall(f"call_{i}", name, {}) for i, name in enumerate(tool_call_names or [])], + tool_results=[], + response={}, + usage=None if usage is _UNSET else usage, + finish_reason=finish_reason, + ) + + +# -- step_count_is --------------------------------------------------------------- + + +def test_step_count_is_false_below_threshold() -> None: + assert step_count_is(2)({"steps": [make_step()]}) is False + + +def test_step_count_is_true_at_threshold_exactly() -> None: + assert step_count_is(2)({"steps": [make_step(), make_step()]}) is True + + +def test_step_count_is_true_when_exceeded() -> None: + assert step_count_is(1)({"steps": [make_step(), make_step(), make_step()]}) is True + + +def test_step_count_is_true_for_zero_steps_with_zero_threshold() -> None: + assert step_count_is(0)({"steps": []}) is True + + +# -- has_tool_call --------------------------------------------------------------- + + +def test_has_tool_call_false_when_no_step_contains_the_tool() -> None: + steps = [make_step(tool_call_names=["search", "browse"])] + assert has_tool_call("email")({"steps": steps}) is False + + +def test_has_tool_call_true_when_any_step_contains_the_tool() -> None: + steps = [make_step(tool_call_names=["search"]), make_step(tool_call_names=["email"])] + assert has_tool_call("email")({"steps": steps}) is True + + +def test_has_tool_call_false_for_empty_steps() -> None: + assert has_tool_call("search")({"steps": []}) is False + + +def test_has_tool_call_accepts_mapping_steps_and_calls() -> None: + # Port-specific: steps may also be plain mappings (camelCase or snake_case). + assert has_tool_call("email")({"steps": [{"toolCalls": [{"name": "email"}]}]}) is True + assert has_tool_call("email")({"steps": [{"tool_calls": [{"name": "search"}]}]}) is False + + +# -- max_tokens_used ------------------------------------------------------------- + + +def test_max_tokens_used_sums_total_tokens_across_steps() -> None: + steps = [make_step(usage={"totalTokens": 400}), make_step(usage={"totalTokens": 700})] + assert max_tokens_used(1000)({"steps": steps}) is True + assert max_tokens_used(1100)({"steps": steps}) is True + assert max_tokens_used(1101)({"steps": steps}) is False + + +def test_max_tokens_used_treats_missing_usage_as_zero() -> None: + steps = [make_step(), make_step(usage=None)] + assert max_tokens_used(1)({"steps": steps}) is False + + +def test_max_tokens_used_treats_missing_total_tokens_as_zero() -> None: + assert max_tokens_used(0)({"steps": [make_step(usage={})]}) is True + + +# -- max_cost -------------------------------------------------------------------- + + +def test_max_cost_sums_cost_across_steps() -> None: + steps = [make_step(usage={"cost": 0.3}), make_step(usage={"cost": 0.25})] + assert max_cost(0.5)({"steps": steps}) is True + assert max_cost(0.6)({"steps": steps}) is False + + +def test_max_cost_treats_missing_cost_as_zero() -> None: + assert max_cost(0.01)({"steps": [make_step(usage={})]}) is False + + +# -- finish_reason_is ------------------------------------------------------------ + + +def test_finish_reason_is_true_when_any_step_has_the_reason() -> None: + steps = [make_step(finish_reason="stop"), make_step(finish_reason="length")] + assert finish_reason_is("length")({"steps": steps}) is True + + +def test_finish_reason_is_false_when_no_step_has_the_reason() -> None: + steps = [make_step(finish_reason="stop"), make_step()] + assert finish_reason_is("length")({"steps": steps}) is False + + +# -- is_stop_condition_met ------------------------------------------------------- + + +async def test_is_stop_condition_met_false_without_conditions() -> None: + assert await is_stop_condition_met([], []) is False + + +async def test_is_stop_condition_met_false_when_all_conditions_false() -> None: + assert await is_stop_condition_met([step_count_is(5), has_tool_call("search")], [make_step()]) is False + + +async def test_is_stop_condition_met_true_when_any_condition_true() -> None: + assert await is_stop_condition_met([step_count_is(5), step_count_is(1)], [make_step()]) is True + + +async def test_is_stop_condition_met_supports_async_conditions() -> None: + async def async_condition(_options: Any) -> bool: + return True + + assert await is_stop_condition_met([async_condition], []) is True + + +async def test_is_stop_condition_met_treats_none_results_as_not_stopping() -> None: + assert await is_stop_condition_met([lambda _options: None], []) is False + + +async def test_is_stop_condition_met_requires_exactly_true() -> None: + # Upstream compares `result === true`; a truthy non-bool does not stop. + assert await is_stop_condition_met([lambda _options: 1, lambda _options: "yes"], []) is False + + +async def test_is_stop_condition_met_evaluates_every_condition_once() -> None: + # Upstream evaluates all conditions with Promise.all — no short-circuit. + calls: List[str] = [] + + def first(_options: Any) -> bool: + calls.append("first") + return True + + async def second(_options: Any) -> bool: + calls.append("second") + return False + + assert await is_stop_condition_met([first, second], []) is True + assert calls == ["first", "second"] + + +async def test_is_stop_condition_met_propagates_failing_conditions() -> None: + async def failing(_options: Any) -> bool: + raise RuntimeError("condition exploded") + + with pytest.raises(RuntimeError, match="condition exploded"): + await is_stop_condition_met([failing], []) diff --git a/tests/unit/test_stop_when_before_approval.py b/tests/unit/test_stop_when_before_approval.py new file mode 100644 index 0000000..34dcb1e --- /dev/null +++ b/tests/unit/test_stop_when_before_approval.py @@ -0,0 +1,124 @@ +"""Port of upstream `tests/unit/stop-when-before-approval.test.ts`. + +With `allow_final_response=False`, a stop_when that is already met on the +first response means no tool from that response runs. It must be evaluated +before the approval check, so pausing for an approval-gated call does not +execute its auto-approved siblings (they are recorded as skipped instead). +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + +from pydantic import BaseModel + +from openrouter_agent import call_model, tool +from openrouter_agent.tool_types import Tool, UnsentToolResult +from tests._fixtures import MemoryStateAccessor, QueuedClient, function_call_item, make_response + + +class VArgs(BaseModel): + v: str + + +class OkOutput(BaseModel): + ok: bool + + +class Counter: + """Counts executions; a class so the return value is explicit (see + tests/unit/test_tool_execution_once.py).""" + + def __init__(self) -> None: + self.count = 0 + + def __call__(self, params: Any, ctx: Any) -> Dict[str, Any]: + self.count += 1 + return {"ok": True} + + +def _make_tools() -> Tuple[List[Tool], Counter, Counter]: + auto_execute = Counter() + gated_execute = Counter() + auto = tool(name="auto", input_schema=VArgs, output_schema=OkOutput, execute=auto_execute) + gated = tool(name="gated", input_schema=VArgs, output_schema=OkOutput, require_approval=True, execute=gated_execute) + return [auto, gated], auto_execute, gated_execute + + +def _client() -> QueuedClient: + return QueuedClient( + [ + make_response( + "resp_1", + [ + function_call_item("call_auto", "auto", '{"v":"x"}'), + function_call_item("call_gated", "gated", '{"v":"x"}'), + ], + ) + ] + ) + + +async def test_does_not_run_auto_sibling_of_gated_call_when_stop_when_already_met() -> None: + tools, auto_execute, gated_execute = _make_tools() + accessor = MemoryStateAccessor() + client = _client() + + result = call_model( + client, + { + "model": "test-model", + "input": "go", + "tools": tools, + "state": accessor, + "stop_when": lambda options: True, + "allow_final_response": False, + }, + ) + await result.get_response() + + assert auto_execute.count == 0 + assert gated_execute.count == 0 + assert len(client.requests) == 1 + + saved = accessor.stored + assert saved is not None + assert saved.status == "awaiting_approval" + assert [call.id for call in saved.pending_tool_calls or []] == ["call_gated"] + # The skipped sibling still gets a matching output for the resume request. + assert saved.unsent_tool_results == [ + UnsentToolResult( + call_id="call_auto", + name="auto", + output=None, + error="Tool execution skipped: stop condition met", + ) + ] + assert await result.requires_approval() is True + + +async def test_still_runs_auto_sibling_when_stop_when_is_not_met() -> None: + tools, auto_execute, gated_execute = _make_tools() + accessor = MemoryStateAccessor() + client = _client() + + await call_model( + client, + { + "model": "test-model", + "input": "go", + "tools": tools, + "state": accessor, + "stop_when": lambda options: False, + "allow_final_response": False, + }, + ).get_response() + + assert auto_execute.count == 1 + assert gated_execute.count == 0 + assert len(client.requests) == 1 + saved = accessor.stored + assert saved is not None + assert saved.status == "awaiting_approval" + assert [call.id for call in saved.pending_tool_calls or []] == ["call_gated"] + assert [result.call_id for result in saved.unsent_tool_results or []] == ["call_auto"] diff --git a/tests/unit/test_tool_choice_relaxation.py b/tests/unit/test_tool_choice_relaxation.py new file mode 100644 index 0000000..30ac189 --- /dev/null +++ b/tests/unit/test_tool_choice_relaxation.py @@ -0,0 +1,365 @@ +"""Port of upstream `tests/unit/tool-choice-relaxation.test.ts` (DEV-785). + +A concrete forced tool choice is one-shot until its resolved semantic value +changes. Reapplying an unchanged choice on every follow-up forbids the model +from ever answering in text, while suppressing a newly resolved dynamic choice +prevents callers from intentionally forcing a later turn. + +Python request key is snake_case `tool_choice`; callable request params are +re-resolved per turn with `ctx["number_of_turns"]`. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Mapping + +import pytest +from pydantic import BaseModel + +from openrouter_agent import call_model, step_count_is, tool +from openrouter_agent.doom_loop import canonicalize_key_material +from tests._fixtures import MemoryStateAccessor, QueuedClient, text_response +from tests._fixtures import tool_call_response as _tool_call_response + + +class WeatherInput(BaseModel): + location: str + + +class WeatherOutput(BaseModel): + temperature: float + + +def _weather_execute(params: Any, ctx: Any) -> Dict[str, Any]: + return {"temperature": 22} + + +def tool_call_response(response_id: str, call_id: str) -> Dict[str, Any]: + return _tool_call_response(response_id, "get_weather", call_id=call_id, arguments='{"location":"Tokyo"}') + + +weather_tool = tool( + name="get_weather", + description="Get the weather for a location.", + input_schema=WeatherInput, + output_schema=WeatherOutput, + execute=_weather_execute, +) + +approval_weather_tool = tool( + name="get_weather", + description="Get the weather for a location.", + input_schema=WeatherInput, + output_schema=WeatherOutput, + execute=_weather_execute, + require_approval=True, +) + +manual_weather_tool = tool( + name="get_weather", + description="Get the weather for a location.", + input_schema=WeatherInput, + output_schema=WeatherOutput, + execute=False, +) + + +def tool_choices(client: QueuedClient) -> List[Any]: + return [request.get("tool_choice") for request in client.requests] + + +async def test_relaxes_required_to_auto_after_a_successful_tool_round() -> None: + client = QueuedClient([tool_call_response("resp_1", "call_1"), text_response("resp_text", "It is 22 degrees.")]) + + text = await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [weather_tool], + "tool_choice": "required", + "stop_when": step_count_is(5), + }, + ).get_text() + + assert text == "It is 22 degrees." + assert len(client.requests) == 2 + # Initial turn: the caller's forced choice reaches the wire intact. + assert client.requests[0]["tool_choice"] == "required" + # Follow-up: relaxed to auto, with tools still available. + assert client.requests[1]["tool_choice"] == "auto" + assert client.requests[1].get("tools") is not None + assert [t["name"] for t in client.requests[1]["tools"]] == ["get_weather"] + + +async def test_relaxes_a_forced_specific_tool_choice_after_a_successful_tool_round() -> None: + client = QueuedClient([tool_call_response("resp_1", "call_1"), text_response("resp_text", "Done.")]) + + text = await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [weather_tool], + "tool_choice": {"type": "function", "name": "get_weather"}, + "stop_when": step_count_is(5), + }, + ).get_text() + + assert text == "Done." + assert len(client.requests) == 2 + assert client.requests[0]["tool_choice"] == {"type": "function", "name": "get_weather"} + assert client.requests[1]["tool_choice"] == "auto" + + +async def test_relaxes_allowed_tools_mode_required_to_mode_auto_keeping_the_tool_set() -> None: + client = QueuedClient([tool_call_response("resp_1", "call_1"), text_response("resp_text", "Done.")]) + allowed_tools = [{"type": "function", "name": "get_weather"}] + + text = await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [weather_tool], + "tool_choice": {"type": "allowed_tools", "mode": "required", "tools": allowed_tools}, + "stop_when": step_count_is(5), + }, + ).get_text() + + assert text == "Done." + assert len(client.requests) == 2 + assert client.requests[0]["tool_choice"] == {"type": "allowed_tools", "mode": "required", "tools": allowed_tools} + assert client.requests[1]["tool_choice"] == {"type": "allowed_tools", "mode": "auto", "tools": allowed_tools} + + +async def test_keeps_tool_choice_auto_unchanged_on_follow_up_turns() -> None: + client = QueuedClient([tool_call_response("resp_1", "call_1"), text_response("resp_text", "Done.")]) + + await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [weather_tool], + "tool_choice": "auto", + "stop_when": step_count_is(5), + }, + ).get_text() + + assert tool_choices(client) == ["auto", "auto"] + + +async def test_still_forces_the_final_no_tools_turn_when_the_budget_is_genuinely_exhausted() -> None: + client = QueuedClient( + [ + tool_call_response("resp_1", "call_1"), + tool_call_response("resp_2", "call_2"), + text_response("resp_text", "Final summary."), + ] + ) + + text = await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [weather_tool], + "tool_choice": "required", + "stop_when": step_count_is(1), + "allow_final_response": True, + }, + ).get_text() + + assert text == "Final summary." + assert len(client.requests) == 3 + assert tool_choices(client) == ["required", "auto", "none"] + + +async def test_keeps_an_unchanged_consumed_choice_relaxed_across_multiple_follow_up_turns() -> None: + client = QueuedClient( + [ + tool_call_response("resp_1", "call_1"), + tool_call_response("resp_2", "call_2"), + text_response("resp_text", "Final summary."), + ] + ) + + text = await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [weather_tool], + "tool_choice": "required", + "stop_when": step_count_is(5), + }, + ).get_text() + + assert text == "Final summary." + assert tool_choices(client) == ["required", "auto", "auto"] + + +async def test_re_arms_when_a_dynamic_forced_choice_resolves_to_a_different_semantic_value() -> None: + client = QueuedClient( + [ + tool_call_response("resp_1", "call_1"), + tool_call_response("resp_2", "call_2"), + text_response("resp_text", "Done."), + ] + ) + seen_turns: List[int] = [] + + def dynamic_choice(ctx: Mapping[str, Any]) -> Any: + seen_turns.append(ctx["number_of_turns"]) + return "required" if ctx["number_of_turns"] == 0 else {"type": "function", "name": "get_weather"} + + text = await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [weather_tool], + "tool_choice": dynamic_choice, + "stop_when": step_count_is(5), + }, + ).get_text() + + assert text == "Done." + assert tool_choices(client) == ["required", {"type": "function", "name": "get_weather"}, "auto"] + # Re-resolved exactly once per dispatched turn. + assert seen_turns == [0, 1, 2] + + +async def test_re_arms_the_same_dynamic_forced_choice_after_an_unforced_turn() -> None: + client = QueuedClient( + [ + tool_call_response("resp_1", "call_1"), + tool_call_response("resp_2", "call_2"), + tool_call_response("resp_3", "call_3"), + text_response("resp_text", "Done."), + ] + ) + + text = await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [weather_tool], + "tool_choice": lambda ctx: "auto" if ctx["number_of_turns"] == 1 else "required", + "stop_when": step_count_is(5), + }, + ).get_text() + + assert text == "Done." + assert tool_choices(client) == ["required", "auto", "required", "auto"] + + +async def test_persists_satisfaction_across_approval_resume_and_resets_it_after_completion() -> None: + accessor = MemoryStateAccessor() + client = QueuedClient( + [ + tool_call_response("resp_1", "call_1"), + text_response("resp_text", "It is 22 degrees."), + tool_call_response("resp_2", "call_2"), + ] + ) + + await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [approval_weather_tool], + "tool_choice": "required", + "state": accessor, + }, + ).get_response() + + assert len(client.requests) == 1 + assert client.requests[0]["tool_choice"] == "required" + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_approval" + assert accessor.stored.consumed_forced_tool_choice_key == canonicalize_key_material("required") + + resumed_text = await call_model( + client, + { + "model": "test-model", + "input": [], + "tools": [approval_weather_tool], + "tool_choice": "required", + "state": accessor, + "approve_tool_calls": ["call_1"], + }, + ).get_text() + + assert resumed_text == "It is 22 degrees." + assert len(client.requests) == 2 + assert client.requests[1]["tool_choice"] == "auto" + assert accessor.stored.status == "complete" + # Upstream asserts `toBeUndefined()`; the Python dataclass field is None. + assert accessor.stored.consumed_forced_tool_choice_key is None + + await call_model( + client, + { + "model": "test-model", + "input": "Start a new weather request.", + "tools": [approval_weather_tool], + "tool_choice": "required", + "state": accessor, + }, + ).get_response() + + assert len(client.requests) == 3 + assert client.requests[2]["tool_choice"] == "required" + + +@pytest.mark.skip( + reason=( + "Not implemented in Python: mid-run interruption via a reloaded state's " + "`interrupted_by` (cancellation signal). Upstream re-loads state during the run and " + "settles to status 'interrupted'; model_result.py loads state once in `_load_state` " + "and has no 'interrupted' status path." + ) +) +async def test_clears_consumed_choice_state_when_an_interrupted_run_becomes_a_fresh_request() -> None: + raise AssertionError("unreachable: skipped") + + +async def test_relaxes_the_first_model_request_after_a_manual_client_tool_resume() -> None: + accessor = MemoryStateAccessor() + client = QueuedClient([tool_call_response("resp_1", "call_1"), text_response("resp_text", "It is 22 degrees.")]) + + await call_model( + client, + { + "model": "test-model", + "input": "What is the weather in Tokyo?", + "tools": [manual_weather_tool], + "tool_choice": "required", + "state": accessor, + }, + ).get_response() + + assert len(client.requests) == 1 + assert accessor.stored is not None + assert accessor.stored.status == "awaiting_client_tools" + assert accessor.stored.consumed_forced_tool_choice_key == canonicalize_key_material("required") + + resumed_text = await call_model( + client, + { + "model": "test-model", + "input": [{"type": "function_call_output", "callId": "call_1", "output": '{"temperature": 22}'}], + "tools": [manual_weather_tool], + "tool_choice": "required", + "state": accessor, + }, + ).get_text() + + assert resumed_text == "It is 22 degrees." + assert len(client.requests) == 2 + assert client.requests[1]["tool_choice"] == "auto" diff --git a/tests/unit/test_tool_context.py b/tests/unit/test_tool_context.py new file mode 100644 index 0000000..634c01a --- /dev/null +++ b/tests/unit/test_tool_context.py @@ -0,0 +1,251 @@ +"""Ports tests/unit/tool-context.test.ts (delta + the ToolContextStore, +buildToolExecuteContext and resolveContext sections). + +Delta at this sync: upstream added two `buildToolExecuteContext` cases about the +per-context never-abort `signal`. This port has no `signal` on the execute +context -- cancellation uses Python-native mechanisms (contract, Idiomatic +Divergences #5) -- so both are skipped as out of scope. The other delta hunk +(`tool()({...})` curried generic) is type-level only. + +Signature note: upstream `buildToolExecuteContext(turnContext, store, toolName, +contextSchema, sharedSchema)`; Python +`build_tool_execute_context(tool, turn_context, store, shared_context_schema)` +reads name and `context_schema` from the tool definition. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +import pytest +from pydantic import BaseModel, ValidationError + +from openrouter_agent import ToolContextStore, tool +from openrouter_agent.tool_context import build_tool_execute_context, resolve_context + +TURN_CONTEXT: Dict[str, Any] = {"number_of_turns": 1} + + +class ApiKey(BaseModel): + apiKey: str + + +class Count(BaseModel): + count: float + + +class Empty(BaseModel): + pass + + +def _tool(name: str, context_schema: Any = None) -> Dict[str, Any]: + return tool(name=name, input_schema=Empty, context_schema=context_schema, execute=lambda params, ctx: None) + + +# -- ToolContextStore ------------------------------------------------------------ + + +def test_store_initializes_with_provided_values() -> None: + store = ToolContextStore({"get_weather": {"apiKey": "sk-123"}, "db_query": {"dbUrl": "postgres://..."}}) + + assert store.get_snapshot() == {"get_weather": {"apiKey": "sk-123"}, "db_query": {"dbUrl": "postgres://..."}} + + +def test_get_snapshot_returns_a_copy() -> None: + store = ToolContextStore({"my_tool": {"key": "value"}}) + + first = store.get_snapshot() + second = store.get_snapshot() + + assert first == second + assert first is not second + + +def test_get_tool_context_by_name() -> None: + store = ToolContextStore({"get_weather": {"apiKey": "sk-123"}}) + + assert store.get_tool_context("get_weather") == {"apiKey": "sk-123"} + assert store.get_tool_context("nonexistent") == {} + + +def test_set_tool_context_notifies_listeners_once() -> None: + store = ToolContextStore({"get_weather": {"apiKey": "old"}}) + snapshots: List[Dict[str, Any]] = [] + store.subscribe(snapshots.append) + + store.set_tool_context("get_weather", {"apiKey": "new"}) + + assert store.get_tool_context("get_weather") == {"apiKey": "new"} + assert snapshots == [{"get_weather": {"apiKey": "new"}}] + + +def test_merge_tool_context_merges_partial_values_and_notifies_once() -> None: + store = ToolContextStore({"my_tool": {"a": 1, "b": 2}}) + snapshots: List[Dict[str, Any]] = [] + store.subscribe(snapshots.append) + + store.merge_tool_context("my_tool", {"b": 3, "c": 4}) + + assert store.get_tool_context("my_tool") == {"a": 1, "b": 3, "c": 4} + assert len(snapshots) == 1 + + +def test_store_supports_unsubscribing() -> None: + store = ToolContextStore({}) + snapshots: List[Dict[str, Any]] = [] + unsubscribe = store.subscribe(snapshots.append) + + store.set_tool_context("my_tool", {"key": "value1"}) + assert len(snapshots) == 1 + + unsubscribe() + store.set_tool_context("my_tool", {"key": "value2"}) + assert len(snapshots) == 1 + + +def test_get_tool_context_returns_a_defensive_copy() -> None: + store = ToolContextStore({"my_tool": {"key": "value"}}) + + returned = store.get_tool_context("my_tool") + returned["key"] = "mutated" + + assert store.get_tool_context("my_tool") == {"key": "value"} + + +# -- buildToolExecuteContext ----------------------------------------------------- + + +@pytest.mark.skip(reason="Out of scope: cancellation signals (no `signal` on the Python execute context).") +def test_gives_each_context_its_own_never_abort_signal() -> None: + raise AssertionError("not ported") + + +@pytest.mark.skip(reason="Out of scope: cancellation signals (no `signal` on the Python execute context).") +def test_prefers_a_caller_supplied_signal_over_a_fresh_one() -> None: + raise AssertionError("not ported") + + +def test_returns_flat_context_with_local_shared_and_set_context() -> None: + store = ToolContextStore({"get_weather": {"apiKey": "sk-123"}}) + + ctx = build_tool_execute_context(_tool("get_weather", ApiKey), TURN_CONTEXT, store) + + # Turn-context fields are merged in. + assert ctx["number_of_turns"] == 1 + assert ctx["local"]["apiKey"] == "sk-123" + assert dict(ctx["shared"]) == {} + assert callable(ctx["set_context"]) + assert callable(ctx["set_shared_context"]) + + +def test_set_context_mutates_the_store() -> None: + store = ToolContextStore({"get_weather": {"apiKey": "old"}}) + ctx = build_tool_execute_context(_tool("get_weather", ApiKey), TURN_CONTEXT, store) + + ctx["set_context"]({"apiKey": "new"}) + + assert store.get_tool_context("get_weather") == {"apiKey": "new"} + # `local` is a live view. + assert ctx["local"]["apiKey"] == "new" + + +@pytest.mark.xfail( + strict=True, + reason=( + "PORT GAP: upstream setContext validates each kept key against the contextSchema shape and " + "throws on mismatch (tool-context.ts:113-133 validatePartialAgainstSchema, z4.parse at :129). " + "Python build_tool_execute_context's set_context (tool_context.py " + "_filter_schema_keys) only filters keys by model_fields and never validates values." + ), +) +def test_set_context_validates_values_against_the_schema() -> None: + store = ToolContextStore({"my_tool": {"count": 0}}) + ctx = build_tool_execute_context(_tool("my_tool", Count), TURN_CONTEXT, store) + + with pytest.raises(ValidationError): + ctx["set_context"]({"count": "not a number"}) + + +def test_set_context_filters_out_keys_not_in_the_schema() -> None: + store = ToolContextStore({"my_tool": {"apiKey": "old", "extra": "extra"}}) + ctx = build_tool_execute_context(_tool("my_tool", ApiKey), TURN_CONTEXT, store) + + ctx["set_context"]({"apiKey": "new", "notInSchema": "ignored"}) + + assert store.get_tool_context("my_tool") == {"apiKey": "new", "extra": "extra"} + + +@pytest.mark.xfail( + strict=True, + reason=( + "PORT GAP: upstream setContext returns early (no-op) when the tool has no contextSchema " + "(tool-context.ts:237-240 `if (!store || !schema) return`; same for setSharedContext). " + "Python _filter_schema_keys (tool_context.py) returns the partial unfiltered when the " + "schema is None, so set_context merges it into the store." + ), +) +def test_set_context_is_a_no_op_when_the_schema_is_undefined() -> None: + store = ToolContextStore({"my_tool": {"key": "value"}}) + ctx = build_tool_execute_context(_tool("my_tool"), TURN_CONTEXT, store) + + ctx["set_context"]({"key": "new"}) + + assert store.get_tool_context("my_tool") == {"key": "value"} + + +def test_returns_empty_context_when_the_store_is_undefined() -> None: + ctx = build_tool_execute_context(_tool("my_tool"), TURN_CONTEXT, None) + + assert ctx["number_of_turns"] == 1 + assert dict(ctx["local"]) == {} + + +def test_set_context_notifies_store_listeners_once() -> None: + store = ToolContextStore({"get_weather": {"apiKey": "old"}}) + ctx = build_tool_execute_context(_tool("get_weather", ApiKey), TURN_CONTEXT, store) + snapshots: List[Dict[str, Any]] = [] + store.subscribe(snapshots.append) + + ctx["set_context"]({"apiKey": "new"}) + + assert len(snapshots) == 1 + assert snapshots[0]["get_weather"] == {"apiKey": "new"} + + +# -- resolveContext -------------------------------------------------------------- + + +async def test_resolve_context_returns_empty_for_none() -> None: + assert await resolve_context(None, {"number_of_turns": 0}) == {} + + +async def test_resolve_context_returns_static_value_as_is() -> None: + value = {"get_weather": {"apiKey": "sk-123"}} + + assert await resolve_context(value, {"number_of_turns": 0}) == value + + +async def test_resolve_context_resolves_a_sync_function() -> None: + def build(turn: Dict[str, Any]) -> Dict[str, Any]: + return {"my_tool": {"turnNum": turn["number_of_turns"]}} + + assert await resolve_context(build, {"number_of_turns": 0}) == {"my_tool": {"turnNum": 0}} + + +async def test_resolve_context_resolves_an_async_function() -> None: + async def build(turn: Dict[str, Any]) -> Dict[str, Any]: + return {"my_tool": {"turnNum": turn["number_of_turns"] + 1}} + + assert await resolve_context(build, {"number_of_turns": 0}) == {"my_tool": {"turnNum": 1}} + + +# -- tool() rejects reserved name "shared" --------------------------------------- + + +def test_tool_rejects_the_reserved_name_shared() -> None: + with pytest.raises(ValueError, match="reserved"): + _tool("shared") + + +def test_tool_accepts_other_names() -> None: + assert _tool("shared_thing")["function"]["name"] == "shared_thing" diff --git a/tests/unit/test_tool_event_broadcaster.py b/tests/unit/test_tool_event_broadcaster.py new file mode 100644 index 0000000..7135526 --- /dev/null +++ b/tests/unit/test_tool_event_broadcaster.py @@ -0,0 +1,354 @@ +"""Ports `packages/agent/tests/unit/tool-event-broadcaster.test.ts`. + +Determinism: upstream's ``setTimeout`` gaps are replaced by explicit hand-offs +(an ``asyncio.Event`` per received item, or :func:`_park`, which yields to the +loop until the consuming task is blocked on its waiter). No wall-clock sleeps. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, AsyncIterator, Dict, List, TypeVar + +import pytest + +from openrouter_agent.tool_event_broadcaster import ToolEventBroadcaster + +T = TypeVar("T") + +_HANG_GUARD_SECONDS = 5 + + +async def _collect(iterator: AsyncIterator[T]) -> List[T]: + return [value async for value in iterator] + + +async def _park(task: "asyncio.Task[Any]") -> None: + for _ in range(10): + await asyncio.sleep(0) + assert not task.done(), "consumer finished before it could be parked" + + +async def test_retains_full_replay_after_completion_by_default() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + first = broadcaster.create_consumer() + + broadcaster.push(1) + broadcaster.complete() + assert await _collect(first) == [1] + + second = broadcaster.create_consumer() + assert await _collect(second) == [1] + + +async def test_compacts_active_consumer_history_at_the_watermark() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster("active-consumers") + first = broadcaster.create_consumer() + + broadcaster.push(1) + broadcaster.push(2) + broadcaster.complete() + assert await first.__anext__() == 1 + second = broadcaster.create_consumer() + assert await _collect(first) == [2] + assert await _collect(second) == [2] + + +# -- single consumer ----------------------------------------------------------- + + +async def test_delivers_events_to_a_single_consumer() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + broadcaster.push(1) + broadcaster.push(2) + broadcaster.push(3) + broadcaster.complete() + + assert await _collect(consumer) == [1, 2, 3] + + +async def test_handles_empty_stream() -> None: + broadcaster: ToolEventBroadcaster[str] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + broadcaster.complete() + + assert await _collect(consumer) == [] + + +async def test_handles_consumer_cancellation_via_aclose() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + broadcaster.push(1) + broadcaster.push(2) + + assert await consumer.__anext__() == 1 + + await consumer.aclose() + + with pytest.raises(StopAsyncIteration): + await consumer.__anext__() + + +# -- multiple consumers -------------------------------------------------------- + + +async def test_delivers_same_events_to_multiple_consumers() -> None: + broadcaster: ToolEventBroadcaster[str] = ToolEventBroadcaster() + consumer1 = broadcaster.create_consumer() + consumer2 = broadcaster.create_consumer() + + broadcaster.push("a") + broadcaster.push("b") + broadcaster.complete() + + results1, results2 = await asyncio.gather(_collect(consumer1), _collect(consumer2)) + + assert results1 == ["a", "b"] + assert results2 == ["a", "b"] + + +async def test_allows_consumers_at_different_read_positions() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer1 = broadcaster.create_consumer() + + broadcaster.push(1) + broadcaster.push(2) + + assert await consumer1.__anext__() == 1 + + # Consumer 2 joins after events were pushed. + consumer2 = broadcaster.create_consumer() + + broadcaster.push(3) + broadcaster.complete() + + # Consumer 1 continues from position 1; consumer 2 replays from 0. + assert await _collect(consumer1) == [2, 3] + assert await _collect(consumer2) == [1, 2, 3] + + +# -- async waiting ------------------------------------------------------------- + + +async def test_waits_for_events_when_consumer_is_ahead_of_buffer() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + consume = asyncio.ensure_future(_collect(consumer)) + + # Push only after the consumer is provably waiting. + await _park(consume) + broadcaster.push(1) + await _park(consume) + broadcaster.push(2) + broadcaster.complete() + + assert await asyncio.wait_for(consume, _HANG_GUARD_SECONDS) == [1, 2] + + +async def test_handles_rapid_push_consume_interleaving() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + received: List[int] = [] + arrived = [asyncio.Event() for _ in range(10)] + + async def consume() -> None: + async for event in consumer: + received.append(event) + arrived[event].set() + + task = asyncio.ensure_future(consume()) + + # Each push is handed off to the consumer before the next one. + for index in range(10): + broadcaster.push(index) + await asyncio.wait_for(arrived[index].wait(), _HANG_GUARD_SECONDS) + broadcaster.complete() + + await asyncio.wait_for(task, _HANG_GUARD_SECONDS) + assert received == list(range(10)) + + +# -- error handling ------------------------------------------------------------ + + +async def test_propagates_errors_to_consumers_after_buffered_events() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + broadcaster.push(1) + broadcaster.complete(RuntimeError("Test error")) + + results: List[int] = [] + with pytest.raises(RuntimeError, match="^Test error$"): + async for event in consumer: + results.append(event) + + assert results == [1] + + +async def test_propagates_errors_to_waiting_consumers() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + consume = asyncio.ensure_future(_collect(consumer)) + await _park(consume) + broadcaster.complete(RuntimeError("Async error")) + + with pytest.raises(RuntimeError, match="Async error"): + await asyncio.wait_for(consume, _HANG_GUARD_SECONDS) + + +async def test_error_alias_completes_with_error() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + broadcaster.error(ValueError("bad")) + assert broadcaster.is_complete is True + with pytest.raises(ValueError, match="bad"): + await consumer.__anext__() + + +# -- ignore after complete ----------------------------------------------------- + + +async def test_ignores_pushes_after_complete() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + broadcaster.push(1) + broadcaster.complete() + broadcaster.push(2) # ignored + + assert await _collect(consumer) == [1] + + +# -- completion between iterations --------------------------------------------- + + +async def test_handles_completion_between_consumer_iterations() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + broadcaster.push(1) + assert await consumer.__anext__() == 1 + + broadcaster.complete() + + with pytest.raises(StopAsyncIteration): + await consumer.__anext__() + + +async def test_handles_completion_with_remaining_buffered_events() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + broadcaster.push(1) + broadcaster.push(2) + broadcaster.push(3) + + assert await consumer.__anext__() == 1 + + broadcaster.complete() + + assert await consumer.__anext__() == 2 + assert await consumer.__anext__() == 3 + with pytest.raises(StopAsyncIteration): + await consumer.__anext__() + + +# -- typed events -------------------------------------------------------------- + + +async def test_works_with_typed_tool_events() -> None: + broadcaster: ToolEventBroadcaster[Dict[str, Any]] = ToolEventBroadcaster() + consumer = broadcaster.create_consumer() + + broadcaster.push({"type": "delta", "content": "test"}) + broadcaster.push({"type": "preliminary_result", "toolCallId": "call_123", "result": {"progress": 50}}) + broadcaster.complete() + + assert await _collect(consumer) == [ + {"type": "delta", "content": "test"}, + {"type": "preliminary_result", "toolCallId": "call_123", "result": {"progress": 50}}, + ] + + +# -- lifecycle compaction (active-consumers) ------------------------------------ + + +async def test_drops_backlog_once_every_consumer_departs_late_joiner_starts_at_watermark() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster("active-consumers") + only = broadcaster.create_consumer() + broadcaster.push(1) + broadcaster.push(2) + assert await only.__anext__() == 1 + assert await only.__anext__() == 2 + await only.aclose() + + # Nobody is attached: retained history is dropped, so a fresh consumer + # starts at the watermark (retention would replay [1, 2] first). + assert broadcaster.find_last_buffered(lambda _value: True) is None + broadcaster.push(3) + broadcaster.complete() + late = broadcaster.create_consumer() + assert await _collect(late) == [3] + + +async def test_backlog_survives_while_at_least_one_consumer_remains() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster("active-consumers") + fast = broadcaster.create_consumer() + slow = broadcaster.create_consumer() + broadcaster.push(1) + broadcaster.push(2) + assert await fast.__anext__() == 1 + assert await fast.__anext__() == 2 + await fast.aclose() + + # The slow consumer still reads from position 0. + assert await slow.__anext__() == 1 + assert await slow.__anext__() == 2 + broadcaster.complete() + with pytest.raises(StopAsyncIteration): + await slow.__anext__() + + +async def test_full_mode_retains_backlog_after_every_consumer_departs() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster() + only = broadcaster.create_consumer() + broadcaster.push(1) + assert await only.__anext__() == 1 + await only.aclose() + broadcaster.complete() + + assert await _collect(broadcaster.create_consumer()) == [1] + + +async def test_aclose_wakes_a_pending_next_instead_of_hanging() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster("active-consumers") + consumer = broadcaster.create_consumer() + broadcaster.push(1) + assert await consumer.__anext__() == 1 + + pending = asyncio.ensure_future(consumer.__anext__()) + await _park(pending) + await consumer.aclose() + with pytest.raises(StopAsyncIteration): + await asyncio.wait_for(pending, _HANG_GUARD_SECONDS) + + +async def test_athrow_rejects_a_pending_next_with_the_same_error() -> None: + broadcaster: ToolEventBroadcaster[int] = ToolEventBroadcaster("active-consumers") + consumer = broadcaster.create_consumer() + broadcaster.push(1) + assert await consumer.__anext__() == 1 + + pending = asyncio.ensure_future(consumer.__anext__()) + await _park(pending) + with pytest.raises(RuntimeError, match="boom"): + await consumer.athrow(RuntimeError("boom")) + with pytest.raises(RuntimeError, match="boom"): + await asyncio.wait_for(pending, _HANG_GUARD_SECONDS) diff --git a/tests/unit/test_tool_executor_return.py b/tests/unit/test_tool_executor_return.py new file mode 100644 index 0000000..53fd045 --- /dev/null +++ b/tests/unit/test_tool_executor_return.py @@ -0,0 +1,232 @@ +"""Ports tests/unit/tool-executor-return.test.ts. + +Upstream (DEV-817) stopped accumulating every yield into a per-call +`preliminaryResults` array: yields are broadcast live through +`onPreliminaryResult` only, and the terminal result no longer carries them. + +DIVERGENCE (contract, Idiomatic Divergences #3): Python async generators cannot +`return` a value. Upstream's "return value" cases are ported as a final +*output-shaped yield*, which is how this port's generator tools deliver their +final result (tool_executor.py `execute_generator_tool`). +""" + +from __future__ import annotations + +from typing import Any, AsyncIterator, Dict, List, Tuple + +from pydantic import BaseModel + +from openrouter_agent import call_model, tool +from openrouter_agent.tool_executor import execute_generator_tool, execute_tool +from tests._fixtures import QueuedClient, text_response, tool_call_response +from openrouter_agent.tool_types import ParsedToolCall + +MOCK_CONTEXT: Dict[str, Any] = {"number_of_turns": 1} + + +class InputParams(BaseModel): + input: str + + +class Empty(BaseModel): + pass + + +class Status(BaseModel): + status: str + + +class ResultOut(BaseModel): + result: str + + +class Index(BaseModel): + index: int + + +class Done(BaseModel): + done: bool + + +class FinalValue(BaseModel): + finalValue: str + + +class Completed(BaseModel): + completed: bool + + +def _no_preliminary_history(result: Dict[str, Any]) -> None: + # Upstream: `expect(result.preliminaryResults).toBeUndefined()`. Check both + # casings so a regression in either naming style is caught. + assert "preliminary_results" not in result + assert "preliminaryResults" not in result + + +async def test_captures_the_final_output_yield_as_the_final_result() -> None: + async def execute(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + yield {"status": "working"} + yield {"status": "almost done"} + # Upstream RETURNs here; Python yields the output-shaped final value. + yield {"result": f"Done: {params.input}"} + + generator_tool = tool( + name="return_value_tool", + input_schema=InputParams, + event_schema=Status, + output_schema=ResultOut, + execute=execute, + ) + tool_call = ParsedToolCall(id="test-call-1", name="return_value_tool", arguments={"input": "test"}) + + result = await execute_generator_tool(generator_tool, tool_call, MOCK_CONTEXT) + + assert result.get("error") is None + _no_preliminary_history(result) + assert result["result"].model_dump() == {"result": "Done: test"} + + +async def test_broadcasts_many_preliminary_results_without_retaining_them_on_the_terminal_result() -> None: + yield_count = 10_000 + + async def execute(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + for index in range(yield_count): + yield {"index": index} + yield {"done": True} + + generator_tool = tool( + name="streaming_tool", + input_schema=Empty, + event_schema=Index, + output_schema=Done, + execute=execute, + ) + tool_call = ParsedToolCall(id="streaming-call", name="streaming_tool", arguments={}) + broadcasts: List[Tuple[str, Any]] = [] + + def on_preliminary_result(call_id: str, value: Any) -> None: + broadcasts.append((call_id, value)) + + result = await execute_generator_tool(generator_tool, tool_call, MOCK_CONTEXT, on_preliminary_result) + + assert len(broadcasts) == yield_count + # Delivered in yield order, each tagged with the executing call's id. + assert {call_id for call_id, _ in broadcasts} == {"streaming-call"} + assert [value.index for _, value in broadcasts[:3]] == [0, 1, 2] + assert broadcasts[-1][1].index == yield_count - 1 + assert result["result"].model_dump() == {"done": True} + _no_preliminary_history(result) + + +async def test_captures_the_final_value_even_with_no_event_yields() -> None: + async def execute(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + # Upstream: no yields, only `return`. Python: one output-shaped yield. + yield {"result": f"Direct: {params.input}"} + + generator_tool = tool( + name="return_only_tool", + input_schema=InputParams, + event_schema=Status, + output_schema=ResultOut, + execute=execute, + ) + tool_call = ParsedToolCall(id="test-call-2", name="return_only_tool", arguments={"input": "test"}) + broadcasts: List[Any] = [] + + result = await execute_generator_tool( + generator_tool, tool_call, MOCK_CONTEXT, lambda call_id, value: broadcasts.append(value) + ) + + assert result.get("error") is None + _no_preliminary_history(result) + assert broadcasts == [] + assert result["result"].model_dump() == {"result": "Direct: test"} + + +async def test_prefers_the_output_shaped_value_over_the_last_event_yield() -> None: + async def execute(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + yield {"status": "progress 1"} + yield {"status": "progress 2"} + yield {"finalValue": f"Final: {params.input}"} + + generator_tool = tool( + name="both_yield_return_tool", + input_schema=InputParams, + event_schema=Status, + output_schema=FinalValue, + execute=execute, + ) + tool_call = ParsedToolCall(id="test-call-3", name="both_yield_return_tool", arguments={"input": "test"}) + broadcasts: List[Any] = [] + + result = await execute_generator_tool( + generator_tool, tool_call, MOCK_CONTEXT, lambda call_id, value: broadcasts.append(value.status) + ) + + assert result.get("error") is None + _no_preliminary_history(result) + assert broadcasts == ["progress 1", "progress 2"] + assert result["result"].model_dump() == {"finalValue": "Final: test"} + + +async def test_still_works_with_yield_only_generators() -> None: + async def execute(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + yield {"status": "working"} + yield {"completed": True} + + generator_tool = tool( + name="yield_only_tool", + input_schema=Empty, + event_schema=Status, + output_schema=Completed, + execute=execute, + ) + tool_call = ParsedToolCall(id="test-call-4", name="yield_only_tool", arguments={}) + + result = await execute_generator_tool(generator_tool, tool_call, MOCK_CONTEXT) + + assert result.get("error") is None + # The last yield matching output_schema but not event_schema is the result. + assert result["result"].model_dump() == {"completed": True} + + +async def test_dispatcher_result_carries_no_preliminary_history() -> None: + async def execute(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + yield {"status": "one"} + yield {"done": True} + + generator_tool = tool(name="gen", input_schema=Empty, event_schema=Status, output_schema=Done, execute=execute) + seen: List[Any] = [] + + result = await execute_tool( + generator_tool, + ParsedToolCall(id="c1", name="gen", arguments={}), + MOCK_CONTEXT, + lambda call_id, value: seen.append((call_id, value.status)), + ) + + assert result is not None + _no_preliminary_history(result) + assert seen == [("c1", "one")] + assert result["result"].model_dump() == {"done": True} + + +async def test_yields_are_delivered_live_through_the_tool_stream_and_absent_from_the_result_event() -> None: + async def execute(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + yield {"status": "one"} + yield {"status": "two"} + yield {"done": True} + + generator_tool = tool(name="gen", input_schema=Empty, event_schema=Status, output_schema=Done, execute=execute) + client = QueuedClient([tool_call_response("resp_1", "gen", call_id="call_gen"), text_response("resp_2", "ok")]) + + result = call_model(client, {"model": "m", "input": "go", "tools": [generator_tool]}) + events = [event async for event in result.get_tool_stream()] + + tool_events = [e for e in events if e["type"] in {"preliminary_result", "tool_result"}] + assert [e["type"] for e in tool_events] == ["preliminary_result", "preliminary_result", "tool_result"] + assert [e["result"].status for e in tool_events[:2]] == ["one", "two"] + assert all(e["toolCallId"] == "call_gen" for e in tool_events) + assert "preliminary_results" not in tool_events[2] + assert "preliminaryResults" not in tool_events[2] + assert len(client.requests) == 2 diff --git a/tests/unit/test_tool_factory_and_executor.py b/tests/unit/test_tool_factory_and_executor.py index 2d53b96..6230ffe 100644 --- a/tests/unit/test_tool_factory_and_executor.py +++ b/tests/unit/test_tool_factory_and_executor.py @@ -77,7 +77,9 @@ async def execute(params: Input, ctx): assert prelim == [("call_2", {"progress": 0.5})] assert result is not None assert result["result"].doubled == 8 - assert result["preliminary_results"] == [{"progress": 0.5}] + # Upstream DEV-817 (tool-executor-return.test.ts): yields are broadcast + # live only; the terminal result no longer carries a yield history. + assert "preliminary_results" not in result async def test_generator_tool_with_overlapping_dict_schemas_keeps_last_yield_as_output() -> None: diff --git a/tests/unit/test_tool_name_events.py b/tests/unit/test_tool_name_events.py new file mode 100644 index 0000000..e33950e --- /dev/null +++ b/tests/unit/test_tool_name_events.py @@ -0,0 +1,233 @@ +"""Ports tests/unit/tool-name-events.test.ts. + +Upstream drives `executeToolsIfNeeded()` on a ModelResult with stubbed private +internals; this port has no such method (the tool loop is inlined in +`ModelResult._run`), so the same events are observed through `call_model`'s +`get_full_responses_stream()` / `get_tool_stream()` (contract, Test Parity). + +Each upstream case is split in two: event type, order, count, `toolCallId`, +`source`, `result`; and the `toolName` field every tool event carries. +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, Sequence + +import pytest +from pydantic import BaseModel + +from openrouter_agent import call_model, tool +from openrouter_agent.tool_types import is_tool_preliminary_result_event, is_tool_result_event +from tests._fixtures import QueuedClient, function_call_item, make_response, text_response + + +class TextIn(BaseModel): + text: str + + +class TextOut(BaseModel): + text: str + + +class Empty(BaseModel): + pass + + +class Stage(BaseModel): + stage: str + + +class Done(BaseModel): + done: bool + + +class Q(BaseModel): + q: str + + +class Answer(BaseModel): + answer: str + + +class Ok(BaseModel): + ok: bool + + +def _dump(value: Any) -> Any: + return value.model_dump() if hasattr(value, "model_dump") else value + + +async def _echo(params: Any, ctx: Any) -> Dict[str, Any]: + return {"text": params.text} + + +async def _progress(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + yield {"stage": "one"} + yield {"stage": "two"} + yield {"done": True} + + +async def _hitl(params: Any, ctx: Any) -> Dict[str, Any]: + return {"answer": "42"} + + +async def _boom(params: Any, ctx: Any) -> Dict[str, Any]: + raise RuntimeError("explode") + + +def echo_tool() -> Dict[str, Any]: + return tool(name="echo", input_schema=TextIn, output_schema=TextOut, execute=_echo) + + +def progress_tool() -> Dict[str, Any]: + return tool(name="progress_tool", input_schema=Empty, event_schema=Stage, output_schema=Done, execute=_progress) + + +def hitl_tool() -> Dict[str, Any]: + return tool(name="hitl_tool", input_schema=Q, output_schema=Answer, on_tool_called=_hitl) + + +def boom_tool() -> Dict[str, Any]: + return tool(name="boom", input_schema=Empty, output_schema=Ok, execute=_boom) + + +SCENARIOS: Dict[str, Dict[str, Any]] = { + "regular": {"tool": echo_tool, "call_id": "call_echo", "name": "echo", "args": {"text": "hi"}}, + "generator": {"tool": progress_tool, "call_id": "call_progress", "name": "progress_tool", "args": {}}, + "hitl": {"tool": hitl_tool, "call_id": "call_hitl", "name": "hitl_tool", "args": {"q": "life?"}}, + "error": {"tool": boom_tool, "call_id": "call_boom", "name": "boom", "args": {}}, +} + + +async def _run(scenario: str) -> List[Any]: + spec = SCENARIOS[scenario] + client = QueuedClient( + [ + make_response("resp_test", [function_call_item(spec["call_id"], spec["name"], json.dumps(spec["args"]))]), + text_response("resp_final", "done"), + ] + ) + result = call_model(client, {"model": "test-model", "input": "hello", "tools": [spec["tool"]()]}) + events = [event async for event in result.get_full_responses_stream()] + # Exactly one tool round then the final turn. + assert len(client.requests) == 2 + return events + + +def _tool_events(events: Sequence[Any]) -> List[Dict[str, Any]]: + return [ + e for e in events if isinstance(e, dict) and (is_tool_result_event(e) or is_tool_preliminary_result_event(e)) + ] + + +# -- parts of each upstream case this port implements ------------------------- + + +async def test_tool_result_for_regular_execute_tools() -> None: + events = await _run("regular") + + tool_results = [e for e in events if isinstance(e, dict) and is_tool_result_event(e)] + assert len(tool_results) == 1 + assert tool_results[0]["type"] == "tool.result" + assert tool_results[0]["toolCallId"] == "call_echo" + assert tool_results[0]["source"] == "client" + assert _dump(tool_results[0]["result"]) == {"text": "hi"} + + +async def test_preliminary_and_final_generator_events() -> None: + events = await _run("generator") + + tool_events = _tool_events(events) + assert [e["type"] for e in tool_events] == [ + "tool.preliminary_result", + "tool.preliminary_result", + "tool.result", + ] + prelims, final = tool_events[:2], tool_events[2] + assert [e["toolCallId"] for e in tool_events] == ["call_progress"] * 3 + assert [_dump(e["result"]) for e in prelims] == [{"stage": "one"}, {"stage": "two"}] + assert _dump(final["result"]) == {"done": True} + assert "preliminaryResults" not in final + assert "preliminary_results" not in final + + +async def test_legacy_tool_stream_projection_for_generator_events() -> None: + client = QueuedClient( + [ + make_response("resp_test", [function_call_item("call_progress", "progress_tool", "{}")]), + text_response("resp_final", "done"), + ] + ) + result = call_model(client, {"model": "test-model", "input": "hello", "tools": [progress_tool()]}) + + projected = [e async for e in result.get_tool_stream() if e["type"] in {"preliminary_result", "tool_result"}] + + assert [e["type"] for e in projected] == ["preliminary_result", "preliminary_result", "tool_result"] + assert [e["toolCallId"] for e in projected] == ["call_progress"] * 3 + + +async def test_tool_result_for_hitl_auto_resolve_path() -> None: + events = await _run("hitl") + + tool_results = [e for e in events if isinstance(e, dict) and is_tool_result_event(e)] + assert len(tool_results) == 1 + assert tool_results[0]["toolCallId"] == "call_hitl" + assert _dump(tool_results[0]["result"]) == {"answer": "42"} + + +async def test_tool_result_for_rejected_or_error_tools() -> None: + events = await _run("error") + + tool_results = [e for e in events if isinstance(e, dict) and is_tool_result_event(e)] + assert len(tool_results) == 1 + assert tool_results[0]["type"] == "tool.result" + assert tool_results[0]["toolCallId"] == "call_boom" + assert tool_results[0]["result"] == {"error": "explode"} + + +def test_preserves_literal_names_on_manual_regular_generator_and_hitl_factory_tools() -> None: + async def gen(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + yield {"ok": True} + + regular = tool(name="alpha", input_schema=Empty, execute=lambda params, ctx: 1) + generator = tool(name="beta", input_schema=Empty, event_schema=Stage, output_schema=Ok, execute=gen) + manual = tool(name="gamma", input_schema=Empty, execute=False) + hitl = tool(name="delta", input_schema=Empty, output_schema=Done, on_tool_called=lambda params, ctx: None) + + assert [t["function"]["name"] for t in (regular, generator, manual, hitl)] == ["alpha", "beta", "gamma", "delta"] + + +# -- toolName --------------------------------------------------------- + + +@pytest.mark.parametrize("scenario", ["regular", "generator", "hitl", "error"]) +async def test_tool_events_carry_tool_name(scenario: str) -> None: + events = await _run(scenario) + + tool_events = _tool_events(events) + expected = SCENARIOS[scenario]["name"] + # Exact event sequence per scenario: the generator emits two preliminary + # results before its single terminal result; the others emit one result. + expected_types = ( + ["tool.preliminary_result", "tool.preliminary_result", "tool.result"] + if scenario == "generator" + else ["tool.result"] + ) + assert [e["type"] for e in tool_events] == expected_types + assert [e.get("toolName") for e in tool_events] == [expected] * len(expected_types) + + +async def test_legacy_tool_stream_projection_carries_tool_name() -> None: + client = QueuedClient( + [ + make_response("resp_test", [function_call_item("call_progress", "progress_tool", "{}")]), + text_response("resp_final", "done"), + ] + ) + result = call_model(client, {"model": "test-model", "input": "hello", "tools": [progress_tool()]}) + + projected = [e async for e in result.get_tool_stream() if e["type"] in {"preliminary_result", "tool_result"}] + + assert len(projected) == 3 + assert [e.get("toolName") for e in projected] == ["progress_tool"] * 3 diff --git a/tests/unit/test_tool_orchestrator.py b/tests/unit/test_tool_orchestrator.py new file mode 100644 index 0000000..39a20de --- /dev/null +++ b/tests/unit/test_tool_orchestrator.py @@ -0,0 +1,512 @@ +"""Ports tests/unit/tool-orchestrator.test.ts. + +Upstream `lib/tool-orchestrator.ts` exports a standalone non-streaming loop +(`executeToolLoop`) plus result helpers (`toolResultsToMap`, +`summarizeToolExecutions`, `hasToolExecutionErrors`, `getToolExecutionErrors`). +This port's `tool_orchestrator.py` is a re-export shim only (see its module +docstring): none of those functions exist. Every upstream case is therefore +ported as a strict xfail against the snake_case API the naming map implies +(`execute_tool_loop`, `tool_results_to_map`, `summarize_tool_executions`, +`has_tool_execution_errors`, `get_tool_execution_errors`), so porting the module +flips them to XPASS. + +The loop invariants that also hold for the inlined `ModelResult._run` loop are +additionally asserted through `call_model` (contract, Test Parity: prefer the +public API) and pass today. +""" + +from __future__ import annotations + +import json +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple + +import pytest +from pydantic import BaseModel + +from openrouter_agent import call_model, mark_mcp, tool, tool_orchestrator +from openrouter_agent.conversation_state import partition_tool_calls +from openrouter_agent.model_result import ModelResult +from openrouter_agent.tool_executor import execute_tool +from tests._fixtures import QueuedClient, function_call_item, make_response, text_response + +# Typed as Any so the missing attributes are a runtime (xfail) failure, not a +# mypy error. +orch: Any = tool_orchestrator + +MISSING_LOOP = ( + "PORT GAP: upstream tool-orchestrator.ts:44 exports executeToolLoop(sendRequest, initialInput, " + "initialRequest, tools, apiTools, options) returning {finalResponse, allResponses, " + "toolExecutionResults, conversationInput}. Python tool_orchestrator.py only re-exports " + "ModelResult/execute_tool/partition_tool_calls; execute_tool_loop does not exist." +) +MISSING_HELPERS = ( + "PORT GAP: upstream tool-orchestrator.ts:185-235 exports toolResultsToMap, " + "summarizeToolExecutions, hasToolExecutionErrors, getToolExecutionErrors. Python " + "tool_orchestrator.py has none of them." +) + + +class AddIn(BaseModel): + a: float + b: float + + +class Empty(BaseModel): + pass + + +class ValueIn(BaseModel): + value: str + + +class Question(BaseModel): + question: str + + +class Pct(BaseModel): + pct: float + + +class Done(BaseModel): + done: bool + + +class ApprovedOut(BaseModel): + approved: bool + + +def _dump(value: Any) -> Any: + return value.model_dump() if hasattr(value, "model_dump") else value + + +def _call(call_id: str, name: str, args: Dict[str, Any]) -> Dict[str, Any]: + return function_call_item(call_id, name, json.dumps(args)) + + +def _response_with(*items: Dict[str, Any], response_id: str = "resp") -> Dict[str, Any]: + return make_response(response_id, list(items)) + + +def _terminal(response_id: str = "resp_end") -> Dict[str, Any]: + return text_response(response_id, "") + + +INITIAL_INPUT: List[Dict[str, Any]] = [{"role": "user", "content": "hi"}] +INITIAL_REQUEST: Dict[str, Any] = {"model": "openai/gpt-4", "input": INITIAL_INPUT} + + +class QueueSendRequest: + """`send_request(input, tools)` stub returning queued responses in order.""" + + def __init__(self, *responses: Dict[str, Any]) -> None: + self._responses = list(responses) + self.calls: List[Tuple[Any, Any]] = [] + + async def __call__(self, input_items: Any, tools: Any) -> Dict[str, Any]: + self.calls.append((input_items, tools)) + index = min(len(self.calls) - 1, len(self._responses) - 1) + return self._responses[index] + + +async def _add(params: Any, ctx: Any) -> Dict[str, Any]: + return {"sum": params.a + params.b} + + +async def _explode(params: Any, ctx: Any) -> Any: + raise RuntimeError("kaboom") + + +async def _progress(params: Any, ctx: Any) -> AsyncIterator[Dict[str, Any]]: + yield {"pct": 50} + yield {"done": True} + + +# -- module surface that exists --------------------------------------------- + + +def test_module_re_exports_the_loop_building_blocks() -> None: + assert tool_orchestrator.ModelResult is ModelResult + assert tool_orchestrator.execute_tool is execute_tool + assert tool_orchestrator.partition_tool_calls is partition_tool_calls + assert sorted(tool_orchestrator.__all__) == ["ModelResult", "execute_tool", "partition_tool_calls"] + + +# -- executeToolLoop (PORT GAP) --------------------------------------------- + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_returns_immediately_without_tool_calls() -> None: + response = _terminal() + send = QueueSendRequest(response) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [], []) + + assert result["final_response"] is response + assert result["all_responses"] == [response] + assert result["tool_execution_results"] == [] + assert result["conversation_input"] is INITIAL_INPUT + assert len(send.calls) == 1 + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_executes_a_tool_call_and_loops_until_done() -> None: + add_tool = tool(name="add", input_schema=AddIn, execute=_add) + first = _response_with(_call("call_1", "add", {"a": 2, "b": 3})) + second = _terminal() + send = QueueSendRequest(first, second) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [add_tool], []) + + assert len(send.calls) == 2 + assert result["all_responses"] == [first, second] + assert result["final_response"] is second + assert len(result["tool_execution_results"]) == 1 + only = result["tool_execution_results"][0] + assert (only["tool_call_id"], only["tool_name"], only["source"]) == ("call_1", "add", "client") + assert _dump(only["result"]) == {"sum": 5} + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_supports_multiple_rounds() -> None: + ping = tool(name="ping", input_schema=Empty, execute=lambda params, ctx: "pong") + send = QueueSendRequest( + _response_with(_call("call_1", "ping", {})), _response_with(_call("call_2", "ping", {})), _terminal() + ) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [ping], []) + + assert len(send.calls) == 3 + assert len(result["all_responses"]) == 3 + assert [r["tool_call_id"] for r in result["tool_execution_results"]] == ["call_1", "call_2"] + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_executes_multiple_calls_in_one_round() -> None: + echo = tool(name="echo", input_schema=ValueIn, execute=lambda params, ctx: params.value) + send = QueueSendRequest( + _response_with(_call("call_a", "echo", {"value": "one"}), _call("call_b", "echo", {"value": "two"})), + _terminal(), + ) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [echo], []) + + assert [r["result"] for r in result["tool_execution_results"]] == ["one", "two"] + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_returns_an_error_result_for_unknown_tools() -> None: + known = tool(name="known", input_schema=Empty, execute=lambda params, ctx: "ok") + send = QueueSendRequest( + _response_with(_call("call_1", "known", {}), _call("call_2", "ghost", {})), + _terminal(), + ) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [known], []) + + by_id = {r["tool_call_id"]: r for r in result["tool_execution_results"]} + assert str(by_id["call_2"]["error"]) == 'Tool "ghost" not found in tool definitions' + assert by_id["call_2"]["source"] == "client" + assert by_id["call_1"].get("error") is None + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_stops_without_executing_manual_tools() -> None: + manual = tool(name="needs_human", input_schema=Question, execute=False) + first = _response_with(_call("call_1", "needs_human", {"question": "?"})) + send = QueueSendRequest(first) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [manual], []) + + assert len(send.calls) == 1 + assert result["tool_execution_results"] == [] + assert result["final_response"] is first + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_captures_execute_failures_and_keeps_looping() -> None: + fail = tool(name="explodes", input_schema=Empty, execute=_explode) + send = QueueSendRequest(_response_with(_call("call_1", "explodes", {})), _terminal()) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [fail], []) + + assert len(result["tool_execution_results"]) == 1 + assert str(result["tool_execution_results"][0]["error"]) == "kaboom" + assert len(send.calls) == 2 + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_marks_mcp_failures_with_source_mcp() -> None: + mcp = mark_mcp(tool(name="mcp_tool", input_schema=Empty, execute=_explode)) + send = QueueSendRequest(_response_with(_call("call_1", "mcp_tool", {})), _terminal()) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [mcp], []) + + only = result["tool_execution_results"][0] + assert only["source"] == "mcp" + assert only["result"] is None + assert str(only["error"]) == "kaboom" + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_filters_hitl_null_results_but_continues() -> None: + hitl = tool(name="approve", input_schema=Empty, output_schema=ApprovedOut, on_tool_called=lambda p, c: None) + send = QueueSendRequest(_response_with(_call("call_1", "approve", {})), _terminal()) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [hitl], []) + + assert result["tool_execution_results"] == [] + assert len(send.calls) == 2 + + +@pytest.mark.skip(reason="Out of scope: background/deferred lifecycle (async) tools are not ported.") +def test_execute_tool_loop_rejects_async_lifecycle_tools_without_invoking_run() -> None: + raise AssertionError("not ported") + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_forwards_generator_preliminary_results() -> None: + progress = tool(name="progress", input_schema=Empty, event_schema=Pct, output_schema=Done, execute=_progress) + send = QueueSendRequest(_response_with(_call("call_1", "progress", {})), _terminal()) + seen: List[Tuple[str, Any]] = [] + + result = await orch.execute_tool_loop( + send, + INITIAL_INPUT, + INITIAL_REQUEST, + [progress], + [], + {"on_preliminary_result": lambda call_id, value: seen.append((call_id, _dump(value)))}, + ) + + assert seen == [("call_1", {"pct": 50})] + assert _dump(result["tool_execution_results"][0]["result"]) == {"done": True} + assert "preliminary_results" not in result["tool_execution_results"][0] + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_applies_next_turn_params_input_changes() -> None: + new_input = [{"role": "user", "content": "rewritten"}] + steer = tool( + name="steer", + input_schema=Empty, + execute=lambda params, ctx: "ok", + next_turn_params={"input": lambda params, ctx: new_input}, + ) + send = QueueSendRequest(_response_with(_call("call_1", "steer", {})), _terminal()) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [steer], []) + + assert result["conversation_input"] == new_input + assert send.calls[-1] == (new_input, []) + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_keeps_original_input_when_next_turn_params_change_nothing() -> None: + plain = tool(name="plain", input_schema=Empty, execute=lambda params, ctx: "ok") + send = QueueSendRequest(_response_with(_call("call_1", "plain", {})), _terminal()) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [plain], []) + + assert result["conversation_input"] is INITIAL_INPUT + + +@pytest.mark.xfail(strict=True, reason=MISSING_LOOP) +async def test_execute_tool_loop_keeps_input_aligned_with_request_when_params_were_computed() -> None: + tune = tool( + name="tune", + input_schema=Empty, + execute=lambda params, ctx: "ok", + next_turn_params={"temperature": lambda params, ctx: 0.2}, + ) + send = QueueSendRequest(_response_with(_call("call_1", "tune", {})), _terminal()) + + result = await orch.execute_tool_loop(send, INITIAL_INPUT, INITIAL_REQUEST, [tune], []) + + assert result["conversation_input"] == INITIAL_REQUEST["input"] + + +# -- result helpers (PORT GAP) ------------------------------------------------ + + +def _result(**overrides: Any) -> Dict[str, Any]: + base: Dict[str, Any] = {"tool_call_id": "call_1", "tool_name": "some_tool", "source": "client", "result": "ok"} + base.update(overrides) + return base + + +@pytest.mark.xfail(strict=True, reason=MISSING_HELPERS) +def test_tool_results_to_map_maps_results_by_tool_call_id() -> None: + mapping = orch.tool_results_to_map([_result(tool_call_id="a", result=1), _result(tool_call_id="b", result=2)]) + + assert mapping == {"a": {"result": 1}, "b": {"result": 2}} + assert len(orch.tool_results_to_map([])) == 0 + + +@pytest.mark.xfail(strict=True, reason=MISSING_HELPERS) +def test_summarize_tool_executions_summarizes_successes_and_errors() -> None: + summary = orch.summarize_tool_executions( + [ + _result(tool_name="good", tool_call_id="a"), + _result(tool_name="chatty", tool_call_id="b"), + _result(tool_name="bad", tool_call_id="c", result=None, error=RuntimeError("nope")), + ] + ) + + assert summary == "\n".join( + ["\u2705 good (a): SUCCESS", "\u2705 chatty (b): SUCCESS", "\u274c bad (c): ERROR - nope"] + ) + assert orch.summarize_tool_executions([]) == "" + + +@pytest.mark.xfail(strict=True, reason=MISSING_HELPERS) +def test_has_tool_execution_errors() -> None: + assert orch.has_tool_execution_errors([_result()]) is False + assert orch.has_tool_execution_errors([]) is False + assert orch.has_tool_execution_errors([_result(), _result(error=RuntimeError("x"), result=None)]) is True + + +@pytest.mark.xfail(strict=True, reason=MISSING_HELPERS) +def test_get_tool_execution_errors_returns_only_errors_in_order() -> None: + boom = RuntimeError("boom") + errors = orch.get_tool_execution_errors( + [ + _result(), + _result(tool_call_id="b", error=boom, result=None), + _result(tool_call_id="c", error=RuntimeError("pow"), result=None), + ] + ) + + assert len(errors) == 2 + assert errors[0] is boom + assert [str(e) for e in errors] == ["boom", "pow"] + assert orch.get_tool_execution_errors([_result()]) == [] + + +# -- the same loop invariants through call_model (pass today) ---------------- + + +def _outputs(request: Dict[str, Any]) -> List[Dict[str, Any]]: + return [i for i in request["input"] if isinstance(i, dict) and i.get("type") == "function_call_output"] + + +async def _tool_results(result: Any) -> List[Dict[str, Any]]: + return [e for e in [ev async for ev in result.get_full_responses_stream()] if _type(e) == "tool.result"] + + +def _type(event: Any) -> Optional[str]: + return event.get("type") if isinstance(event, dict) else getattr(event, "type", None) + + +async def test_loop_returns_after_one_request_without_tool_calls() -> None: + client = QueuedClient([text_response("resp_1", "hello")]) + + text = await call_model(client, {"model": "m", "input": INITIAL_INPUT, "tools": []}).get_text() + + assert text == "hello" + assert len(client.requests) == 1 + + +async def test_loop_executes_a_call_and_sends_its_output_on_the_follow_up() -> None: + add_tool = tool(name="add", input_schema=AddIn, execute=_add) + client = QueuedClient([_response_with(_call("call_1", "add", {"a": 2, "b": 3})), text_response("r2", "5")]) + + result = call_model(client, {"model": "m", "input": INITIAL_INPUT, "tools": [add_tool]}) + tool_results = await _tool_results(result) + + assert len(client.requests) == 2 + assert [(e["toolCallId"], e["source"], e["result"]) for e in tool_results] == [("call_1", "client", {"sum": 5})] + outputs = _outputs(client.requests[1]) + # snake_case `call_id`: _send normalizes at the transport boundary + # (model_result.py `_send`); upstream's wire item uses `callId`. + assert [(o["call_id"], json.loads(o["output"])) for o in outputs] == [("call_1", {"sum": 5})] + + +async def test_loop_supports_multiple_rounds_executing_each_call_once() -> None: + calls: List[str] = [] + + def ping(params: Any, ctx: Any) -> str: + calls.append(ctx["tool_call"]["callId"]) + return "pong" + + ping_tool = tool(name="ping", input_schema=Empty, execute=ping) + client = QueuedClient( + [ + _response_with(_call("call_1", "ping", {}), response_id="r1"), + _response_with(_call("call_2", "ping", {}), response_id="r2"), + text_response("r3", "end"), + ] + ) + + await call_model(client, {"model": "m", "input": INITIAL_INPUT, "tools": [ping_tool]}).get_text() + + assert len(client.requests) == 3 + assert calls == ["call_1", "call_2"] + + +async def test_loop_executes_multiple_calls_in_one_round_in_order() -> None: + echo = tool(name="echo", input_schema=ValueIn, execute=lambda params, ctx: params.value) + client = QueuedClient( + [ + _response_with(_call("call_a", "echo", {"value": "one"}), _call("call_b", "echo", {"value": "two"})), + text_response("r2", "end"), + ] + ) + + await call_model(client, {"model": "m", "input": INITIAL_INPUT, "tools": [echo]}).get_text() + + assert len(client.requests) == 2 + outputs = _outputs(client.requests[1]) + assert [(o["call_id"], json.loads(o["output"])) for o in outputs] == [("call_a", "one"), ("call_b", "two")] + + +async def test_loop_stops_without_executing_manual_tools() -> None: + manual = tool(name="needs_human", input_schema=Question, execute=False) + first = _response_with(_call("call_1", "needs_human", {"question": "?"})) + client = QueuedClient([first]) + + result = call_model(client, {"model": "m", "input": INITIAL_INPUT, "tools": [manual]}) + final = await result.get_response() + + assert len(client.requests) == 1 + assert final is first + assert await _tool_results(result) == [] + + +async def test_loop_captures_execute_failures_and_keeps_looping() -> None: + fail = tool(name="explodes", input_schema=Empty, execute=_explode) + client = QueuedClient([_response_with(_call("call_1", "explodes", {})), text_response("r2", "end")]) + + result = call_model(client, {"model": "m", "input": INITIAL_INPUT, "tools": [fail]}) + tool_results = await _tool_results(result) + + assert [e["result"] for e in tool_results] == [{"error": "kaboom"}] + assert len(client.requests) == 2 + outputs = _outputs(client.requests[1]) + assert [json.loads(o["output"]) for o in outputs] == [{"error": "kaboom"}] + + +async def test_loop_marks_mcp_failures_with_source_mcp() -> None: + mcp = mark_mcp(tool(name="mcp_tool", input_schema=Empty, execute=_explode)) + client = QueuedClient([_response_with(_call("call_1", "mcp_tool", {})), text_response("r2", "end")]) + + result = call_model(client, {"model": "m", "input": INITIAL_INPUT, "tools": [mcp]}) + tool_results = await _tool_results(result) + + assert [(e["source"], e["result"]) for e in tool_results] == [("mcp", {"error": "kaboom"})] + + +async def test_loop_forwards_generator_preliminary_results_before_the_result() -> None: + progress = tool(name="progress", input_schema=Empty, event_schema=Pct, output_schema=Done, execute=_progress) + client = QueuedClient([_response_with(_call("call_1", "progress", {})), text_response("r2", "end")]) + + result = call_model(client, {"model": "m", "input": INITIAL_INPUT, "tools": [progress]}) + events = [ + e + for e in [ev async for ev in result.get_full_responses_stream()] + if _type(e) in {"tool.preliminary_result", "tool.result"} + ] + + assert [_type(e) for e in events] == ["tool.preliminary_result", "tool.result"] + assert _dump(events[0]["result"]) == {"pct": 50} + assert _dump(events[1]["result"]) == {"done": True} + assert "preliminary_results" not in events[1] diff --git a/tests/unit/test_tool_set.py b/tests/unit/test_tool_set.py new file mode 100644 index 0000000..4af0fb5 --- /dev/null +++ b/tests/unit/test_tool_set.py @@ -0,0 +1,616 @@ +"""Ports `packages/agent/tests/unit/tool-set.test.ts`. + +Runtime cases are ported 1:1 (same `describe` grouping, as classes). Upstream +cases that are *purely* `expectTypeOf` assertions are not reproducible in +Python (contract divergence 4 -- typing is looser) and are omitted: + +- clone: "widens the partition/situation types when cloning to mutable", + "preserves the exact source partition type on clone() and clone({mutable: false})" +- mutable aliasing soundness: "gives createToolSet({mutable: true}) the widened + partition/situation types" +- compile-time partition inference: "tracks static activate/deactivate + transitions", "moves IDs into conditional via activateWhen/deactivateWhen" +- defineSituations: "keeps situation names literal at compile time" +- InferToolSet / event narrowing: "aliases CorrelatedToolEventUnion" + +Mixed type/runtime cases keep only their runtime assertions. + +Server-tool ids: upstream `serverTool(config, { id })` stamps `id` on the tool +(`src/lib/tool.ts` serverTool). This port's `server_tool` has no `id` option, +so custom-id server tools are built as `{**server_tool(cfg), "id": ...}` and the +default id is read via `get_tool_id` (same `server:${type}` rule). +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +import pytest + +from openrouter_agent.tool import server_tool, tool +from openrouter_agent.tool_set import ( + TOOL_SET_SNAPSHOT, + ActivationInput, + ToolSet, + create_tool_set, + get_tool_id, +) +from openrouter_agent.tool_types import ConversationState + + +def make_tool(name: str) -> Dict[str, Any]: + async def execute(params: Any, ctx: Any) -> Dict[str, Any]: + return {"name": name} + + return tool(name=name, description=f"{name} tool", input_schema=dict, execute=execute) + + +a = make_tool("a") +b = make_tool("b") +c = make_tool("c") + + +def minimal_state(**partial: Any) -> ConversationState: + base: Dict[str, Any] = {"id": "conv_test", "messages": [], "status": "complete", "created_at": 0, "updated_at": 0} + base.update(partial) + return ConversationState(**base) + + +def ctx(inp: ActivationInput) -> Dict[str, Any]: + return dict(inp.get("context") or {}) + + +class PredicateSpy: + def __init__(self, result: bool = True) -> None: + self.calls: List[ActivationInput] = [] + self.result = result + + def __call__(self, inp: ActivationInput) -> bool: + self.calls.append(inp) + return self.result + + +class TestCreateToolSet: + def test_preserves_tool_order_via_tools(self) -> None: + ts = create_tool_set([a, b, c]) + assert [get_tool_id(t) for t in ts.tools] == ["a", "b", "c"] + assert ts.tools == (a, b, c) + + def test_throws_on_duplicate_tool_names_at_construction(self) -> None: + dup = make_tool("a") + with pytest.raises(ValueError, match='Duplicate tool ID: "a"'): + create_tool_set([a, dup]) + + def test_constructs_an_empty_set_without_tools(self) -> None: + ts = create_tool_set([]) + assert ts.tools == () + resolved = ts.resolve() + assert resolved.tools == [] + assert resolved.active_tools == [] + assert resolved.enabled == [] + assert resolved.disabled == [] + assert resolved.status_by_tool == {} + + def test_defaults_all_tools_to_active_when_no_directives_are_set(self) -> None: + resolved = create_tool_set([a, b]).resolve() + assert resolved.tools == [a, b] + assert resolved.active_tools == ["a", "b"] + assert resolved.enabled == ["a", "b"] + assert resolved.disabled == [] + assert resolved.status_by_tool == { + "a": {"enabled": True, "reason": "default"}, + "b": {"enabled": True, "reason": "default"}, + } + + +class TestActivateDeactivate: + def test_deactivates_a_single_tool_by_name(self) -> None: + ts = create_tool_set([a, b, c]).deactivate("b") + assert ts.resolve().active_tools == ["a", "c"] + assert ts.resolve().disabled == ["b"] + + def test_activates_deactivates_arrays_of_names(self) -> None: + ts = create_tool_set([a, b, c]).deactivate(["a", "b"]).activate(["b"]) + assert ts.resolve().active_tools == ["b", "c"] + + def test_throws_on_unknown_names(self) -> None: + ts = create_tool_set([a]) + with pytest.raises(ValueError, match='Unknown tool: "missing"'): + ts.activate("missing") + with pytest.raises(ValueError, match='Unknown tool: "missing"'): + ts.deactivate(["a", "missing"]) + + +class TestActivateWhen: + def test_defaults_to_inactive_and_flips_based_on_predicate(self) -> None: + ts = create_tool_set([a, b]).activate_when("a", lambda i: ctx(i).get("enabled") is True) + assert ts.resolve().active_tools == ["b"] + assert ts.resolve({"context": {"enabled": True}}).active_tools == ["a", "b"] + + def test_accepts_a_predicate_map(self) -> None: + ts = create_tool_set([a, b]).activate_when({"a": lambda _i: True, "b": lambda _i: False}) + assert ts.resolve().active_tools == ["a"] + + def test_validates_every_name_in_the_map_before_applying(self) -> None: + ts = create_tool_set([a, b]) + with pytest.raises(ValueError, match='Unknown tool: "nope"'): + ts.activate_when({"a": lambda _i: True, "nope": lambda _i: True}) + # original untouched + assert ts.resolve().active_tools == ["a", "b"] + + +class TestDeactivateWhen: + def test_defaults_to_active_and_flips_inactive_when_predicate_is_true(self) -> None: + ts = create_tool_set([a, b]).deactivate_when("a", lambda _i: True) + assert ts.resolve().active_tools == ["b"] + + def test_accepts_a_predicate_map(self) -> None: + ts = create_tool_set([a, b]).deactivate_when({"a": lambda _i: True, "b": lambda _i: False}) + assert ts.resolve().active_tools == ["b"] + + +class TestLastCallWins: + def test_resolves_to_the_most_recent_directive_per_tool(self) -> None: + ts = create_tool_set([a, b]).activate("a").deactivate_when("a", lambda _i: True) + assert ts.resolve().active_tools == ["b"] + + ts2 = create_tool_set([a, b]).deactivate_when("a", lambda _i: True).activate("a") + assert ts2.resolve().active_tools == ["a", "b"] + + +class TestImmutabilityVsMutability: + def test_is_immutable_by_default_mutators_return_a_new_instance(self) -> None: + base = create_tool_set([a, b]) + nxt = base.deactivate("a") + assert nxt is not base + assert base.resolve().active_tools == ["a", "b"] + assert nxt.resolve().active_tools == ["b"] + + def test_mutates_in_place_when_mutable_true(self) -> None: + base = create_tool_set([a, b], mutable=True) + nxt = base.deactivate("a") + assert nxt is base + assert base.resolve().active_tools == ["b"] + + +class TestClone: + def test_copies_state_and_can_flip_mode(self) -> None: + immutable = create_tool_set([a, b]).deactivate("a") + mutable_copy = immutable.clone(mutable=True) + mutable_copy.activate("a") + assert mutable_copy.resolve().active_tools == ["a", "b"] + # original untouched + assert immutable.resolve().active_tools == ["b"] + + def test_inherits_mode_when_not_overridden(self) -> None: + mutable = create_tool_set([a], mutable=True) + clone = mutable.clone() + after = clone.deactivate("a") + assert after is clone + # A clone is independent of its source. + assert mutable.resolve().active_tools == ["a"] + + def test_clone_mutable_false_from_mutable_returns_new_instances(self) -> None: + # Runtime half of upstream "preserves the exact source partition type + # on clone() and clone({mutable: false})". + mutable = create_tool_set([a, b], mutable=True) + immutable = mutable.clone(mutable=False) + after = immutable.deactivate("a") + assert after is not immutable + assert immutable.resolve().active_tools == ["a", "b"] + assert after.resolve().active_tools == ["b"] + + +class TestMutableAliasingSoundness: + def test_every_alias_of_a_mutable_instance_is_the_same_object(self) -> None: + base = create_tool_set([a, b, c], mutable=True) + alias_one = base + alias_two = base + + after_activate = alias_one.activate("a") + after_deactivate = alias_two.deactivate("b") + after_activate_when = after_activate.activate_when("c", lambda _i: True) + after_deactivate_when = after_deactivate.deactivate_when("c", lambda _i: False) + + assert after_activate is base + assert after_deactivate is base + assert after_activate_when is base + assert after_deactivate_when is base + + def test_mutable_alias_observes_mutations_made_through_another_alias(self) -> None: + mutable = create_tool_set([a, b], mutable=True) + alias = mutable + mutable.deactivate("a") + assert alias.resolve().active_tools == ["b"] + + def test_keeps_the_immutable_chain_producing_distinct_instances(self) -> None: + base = create_tool_set([a, b, c]) + after_deactivate = base.deactivate("b") + after_activate_when = after_deactivate.activate_when("a", lambda _i: True) + assert after_deactivate is not base + assert after_activate_when is not after_deactivate + + +class TestResolveInputShapes: + def test_handles_none_and_empty_input(self) -> None: + ts = create_tool_set([a]).activate_when("a", lambda i: "state" not in i and "context" not in i) + assert ts.resolve().active_tools == ["a"] + assert ts.resolve({}).active_tools == ["a"] + + def test_passes_state_and_context_to_the_predicate(self) -> None: + spy = PredicateSpy() + ts = create_tool_set([a]).activate_when("a", spy) + state = minimal_state(messages=[{"role": "user", "content": "hi"}]) + ts.resolve({"state": state, "context": {"foo": "bar"}}) + assert spy.calls == [{"state": state, "context": {"foo": "bar"}}] + + def test_keeps_infer_tools_as_a_back_compat_alias_of_resolve(self) -> None: + ts = create_tool_set([a, b]).deactivate("a") + via_resolve = ts.resolve() + via_infer = ts.infer_tools() + assert via_infer["tools"] == via_resolve.tools + assert via_infer["active_tools"] == via_resolve.active_tools + assert via_infer["enabled"] == via_resolve.enabled + assert via_infer["disabled"] == via_resolve.disabled + assert via_infer["status_by_tool"] == via_resolve.status_by_tool + # Upstream marks the result with the TOOL_SET_SNAPSHOT symbol so + # call_model can strip its metadata when the whole object is spread. + assert via_infer[TOOL_SET_SNAPSHOT] is True + assert {**via_infer}[TOOL_SET_SNAPSHOT] is True + + +class TestExhaustiveStatusByTool: + def test_includes_every_id_with_reason_directive_predicate_metadata(self) -> None: + ts = create_tool_set([a, b, c]).deactivate("b").activate_when("c", lambda _i: True) + resolved = ts.resolve() + assert sorted(resolved.status_by_tool) == ["a", "b", "c"] + assert resolved.status_by_tool["a"] == {"enabled": True, "reason": "default"} + assert resolved.status_by_tool["b"] == {"enabled": False, "reason": "deactivate", "directive": "deactivate"} + assert resolved.status_by_tool["c"] == { + "enabled": True, + "reason": "activateWhen", + "directive": "activateWhen", + "predicate": True, + } + assert resolved.enabled == ["a", "c"] + assert resolved.disabled == ["b"] + + def test_keeps_prototype_sensitive_ids_as_real_keys(self) -> None: + # Upstream guards JS `__proto__` setter semantics; Python dicts have no + # such hazard, but the invariant (every id is a real key, nothing + # dropped) is still worth pinning. + dunder_proto = make_tool("__proto__") + ctor = make_tool("constructor") + proto = make_tool("prototype") + + resolved = create_tool_set([a, dunder_proto, ctor, proto, b]).deactivate("constructor").resolve() + + assert sorted(resolved.status_by_tool) == ["__proto__", "a", "b", "constructor", "prototype"] + assert resolved.status_by_tool["constructor"] == { + "enabled": False, + "reason": "deactivate", + "directive": "deactivate", + } + assert resolved.status_by_tool["prototype"] == {"enabled": True, "reason": "default"} + assert resolved.status_by_tool["__proto__"] == {"enabled": True, "reason": "default"} + assert resolved.status_by_tool["a"] == {"enabled": True, "reason": "default"} + assert resolved.status_by_tool["b"] == {"enabled": True, "reason": "default"} + + assert resolved.enabled == ["a", "__proto__", "prototype", "b"] + assert resolved.disabled == ["constructor"] + assert resolved.tools == [a, dunder_proto, proto, b] + assert resolved.active_tools == ["a", "__proto__", "prototype", "b"] + + def test_resolves_exotic_ids_individually(self) -> None: + dunder_proto = make_tool("__proto__") + ctor = make_tool("constructor") + proto = make_tool("prototype") + + resolved = ( + create_tool_set([dunder_proto, ctor, proto]) + .activate("__proto__") + .deactivate("prototype") + .activate_when("constructor", lambda _i: False) + .resolve() + ) + + assert resolved.status_by_tool["__proto__"] == {"enabled": True, "reason": "activate", "directive": "activate"} + assert resolved.status_by_tool["constructor"] == { + "enabled": False, + "reason": "activateWhen", + "directive": "activateWhen", + "predicate": True, + } + assert resolved.status_by_tool["prototype"] == { + "enabled": False, + "reason": "deactivate", + "directive": "deactivate", + } + assert sorted(resolved.enabled) == ["__proto__"] + assert sorted(resolved.disabled) == ["constructor", "prototype"] + + +class TestPartitionRuntime: + def test_includes_a_conditional_id_in_disabled_when_predicate_resolves_false(self) -> None: + snapshot = create_tool_set([a, b]).deactivate("b").activate_when("a", lambda _i: False).resolve() + assert snapshot.disabled == ["a", "b"] + + def test_returns_the_active_tools_for_static_partitions(self) -> None: + resolved = create_tool_set([a, b, c]).deactivate("b").resolve() + assert resolved.tools == [a, c] + assert resolved.active_tools == ["a", "c"] + + def test_predicate_must_return_exactly_true(self) -> None: + # Upstream compares `predicate(input) === true`; truthy non-bools do not count. + resolved = ( + create_tool_set([a, b]).activate_when("a", lambda _i: 1).deactivate_when("b", lambda _i: "yes").resolve() + ) + assert resolved.enabled == ["b"] + assert resolved.disabled == ["a"] + + +class TestServerTools: + web_search = server_tool({"type": "web_search_2025_08_26"}) + datetime = server_tool({"type": "openrouter:datetime"}) + # Upstream: serverTool({type}, {id: 'server:public_search'}). + public_search = {**server_tool({"type": "web_search_2025_08_26"}), "id": "server:public_search"} + + def test_assigns_default_server_ids_from_config_type(self) -> None: + assert get_tool_id(self.web_search) == "server:web_search_2025_08_26" + assert get_tool_id(self.datetime) == "server:openrouter:datetime" + assert get_tool_id(self.public_search) == "server:public_search" + + def test_server_tool_stamps_default_id(self) -> None: + assert self.web_search.get("id") == "server:web_search_2025_08_26" + + def test_preserves_server_tools_in_tools_in_construction_order(self) -> None: + ts = create_tool_set([a, self.web_search, b, self.datetime]) + assert ts.tools == (a, self.web_search, b, self.datetime) + assert ts.resolve().enabled == ["a", "server:web_search_2025_08_26", "b", "server:openrouter:datetime"] + + def test_includes_active_server_tools_in_tools_enabled_status_but_not_active_tools(self) -> None: + resolved = create_tool_set([a, self.web_search, b]).deactivate("a").resolve() + assert resolved.tools == [self.web_search, b] + assert resolved.active_tools == ["b"] + assert resolved.enabled == ["server:web_search_2025_08_26", "b"] + assert resolved.status_by_tool["server:web_search_2025_08_26"] == {"enabled": True, "reason": "default"} + + def test_can_deactivate_server_tools_by_stable_id(self) -> None: + resolved = create_tool_set([a, self.web_search, b]).deactivate("server:web_search_2025_08_26").resolve() + assert resolved.tools == [a, b] + assert resolved.enabled == ["a", "b"] + assert resolved.disabled == ["server:web_search_2025_08_26"] + assert resolved.status_by_tool["server:web_search_2025_08_26"] == { + "enabled": False, + "reason": "deactivate", + "directive": "deactivate", + } + + def test_supports_override_ids_and_rejects_duplicates(self) -> None: + ts = create_tool_set([a, self.public_search]) + assert ts.resolve().enabled == ["a", "server:public_search"] + + with pytest.raises(ValueError, match='Duplicate tool ID: "server:web_search_2025_08_26"'): + create_tool_set([self.web_search, server_tool({"type": "web_search_2025_08_26"})]) + + def test_rejects_activate_on_raw_type_strings(self) -> None: + ts = create_tool_set([a, self.web_search]) + with pytest.raises(ValueError, match="Unknown tool"): + ts.activate("web_search_2025_08_26") + + def test_hand_written_server_tool_without_id_uses_synthesized_id(self) -> None: + hand_written = {"_brand": "server-tool", "config": {"type": "web_search_2025_08_26"}} + tid = "server:web_search_2025_08_26" + resolved = create_tool_set([a, hand_written]).deactivate(tid).resolve() + assert resolved.enabled == ["a"] + assert resolved.disabled == [tid] + assert resolved.status_by_tool[tid] == {"enabled": False, "reason": "deactivate", "directive": "deactivate"} + assert resolved.tools == [a] + + def test_empty_explicit_id_falls_back_to_synthesized_id(self) -> None: + # Upstream defaultServerId: only a non-empty string id wins. + empty_id = {**server_tool({"type": "openrouter:datetime"}), "id": ""} + assert create_tool_set([empty_id]).resolve().enabled == ["server:openrouter:datetime"] + + def test_custom_id_server_tool_accepts_the_real_runtime_id(self) -> None: + resolved = create_tool_set([a, self.public_search]).deactivate("server:public_search").resolve() + assert resolved.enabled == ["a"] + assert resolved.disabled == ["server:public_search"] + + def test_custom_id_server_tool_rejects_the_synthesized_default_id(self) -> None: + ts = create_tool_set([a, self.public_search]) + with pytest.raises(ValueError, match="Unknown tool"): + ts.deactivate("server:web_search_2025_08_26") + + +class TestSituations: + def test_overlays_static_enabled_disabled(self) -> None: + ts = ( + create_tool_set([a, b, c]) + .deactivate("c") + .define_situations( + { + "guest": {"enabled": ["a"], "disabled": ["b", "c"]}, + "full": {"enabled": ["a", "b", "c"]}, + } + ) + ) + + guest = ts.resolve_situation("guest") + assert guest.tools == [a] + assert guest.active_tools == ["a"] + assert guest.enabled == ["a"] + assert guest.disabled == ["b", "c"] + assert guest.status_by_tool == { + "a": {"enabled": True, "reason": "situation", "directive": "activate"}, + "b": {"enabled": False, "reason": "situation", "directive": "deactivate"}, + "c": {"enabled": False, "reason": "situation", "directive": "deactivate"}, + } + + full = ts.resolve_situation("full") + assert full.tools == [a, b, c] + + def test_supports_conditional_situation_rules_with_runtime_exact_status(self) -> None: + ts = create_tool_set([a, b, c]).define_situations( + { + "authed": { + "enabled": ["a"], + "disabled": ["b"], + "conditional": {"c": lambda i: ctx(i).get("admin") is True}, + } + } + ) + + denied = ts.resolve_situation("authed", {"context": {"admin": False}}) + assert denied.tools == [a] + assert denied.enabled == ["a"] + assert denied.disabled == ["b", "c"] + assert denied.status_by_tool["c"] == { + "enabled": False, + "reason": "situation", + "directive": "activateWhen", + "predicate": True, + } + + allowed = ts.resolve_situation("authed", {"context": {"admin": True}}) + assert allowed.tools == [a, c] + assert allowed.enabled == ["a", "c"] + + def test_supports_deactivate_when_situation_rules(self) -> None: + ts = create_tool_set([a]).define_situations( + { + "guarded": { + "conditional": { + "a": {"mode": "deactivateWhen", "predicate": lambda i: ctx(i).get("blocked") is True} + } + } + } + ) + + assert ts.resolve_situation("guarded").enabled == ["a"] + blocked = ts.resolve_situation("guarded", {"context": {"blocked": True}}) + assert blocked.disabled == ["a"] + assert blocked.status_by_tool["a"] == { + "enabled": False, + "reason": "situation", + "directive": "deactivateWhen", + "predicate": True, + } + + def test_validates_unknown_duplicate_conflicting_ids(self) -> None: + base = create_tool_set([a, b]) + + with pytest.raises(ValueError, match='Unknown tool: "nope"'): + base.define_situations({"bad": {"enabled": ["nope"]}}) + + with pytest.raises(ValueError, match='lists tool "a" more than once'): + base.define_situations({"bad": {"enabled": ["a"], "disabled": ["a"]}}) + + with pytest.raises(ValueError, match='lists tool "a" more than once'): + base.define_situations({"bad": {"enabled": ["a"], "conditional": {"a": lambda _i: True}}}) + + def test_ignores_none_conditional_entries(self) -> None: + ts = create_tool_set([a, b]).define_situations( + {"optional": {"conditional": {"a": None, "b": lambda _i: False}}} + ) + assert ts.resolve_situation("optional").active_tools == ["a"] + + def test_rejects_malformed_conditional_rules_at_definition_time(self) -> None: + base = create_tool_set([a]) + expected = ( + 'Situation "checkout": conditional rule for tool "a" must be a function or { mode, predicate } object' + ) + with pytest.raises(ValueError) as excinfo: + base.define_situations({"checkout": {"conditional": {"a": "invalid"}}}) # type: ignore[dict-item] + assert str(excinfo.value) == expected + + def test_rejects_a_missing_conditional_predicate_at_definition_time(self) -> None: + base = create_tool_set([a]) + expected = ( + 'Situation "checkout": conditional rule for tool "a" must be a function or { mode, predicate } object' + ) + with pytest.raises(ValueError) as excinfo: + base.define_situations({"checkout": {"conditional": {"a": {"mode": "activateWhen"}}}}) + assert str(excinfo.value) == expected + + def test_rejects_an_unknown_conditional_mode(self) -> None: + base = create_tool_set([a]) + with pytest.raises(ValueError, match='Situation "checkout": conditional rule for tool "a"'): + base.define_situations( + {"checkout": {"conditional": {"a": {"mode": "sometimes", "predicate": lambda _i: True}}}} # type: ignore[typeddict-item, dict-item] + ) + + def test_throws_on_unknown_situation_names_at_resolve_time(self) -> None: + ts = create_tool_set([a]).define_situations({"guest": {"enabled": ["a"]}}) + with pytest.raises(ValueError, match='Unknown situation: "missing"'): + ts.resolve_situation("missing") + + def test_leaves_unmentioned_ids_on_the_base_partition(self) -> None: + ts = create_tool_set([a, b, c]).deactivate("c").define_situations({"onlyB": {"disabled": ["b"]}}) + snapshot = ts.resolve_situation("onlyB") + # a stays default-enabled, b disabled by situation, c disabled by base + assert snapshot.enabled == ["a"] + assert snapshot.disabled == ["b", "c"] + + def test_define_situations_replaces_the_registry_and_respects_mutability(self) -> None: + # Upstream docstring: "Replaces any previously defined situations". + immutable = create_tool_set([a]).define_situations({"one": {"disabled": ["a"]}}) + replaced = immutable.define_situations({"two": {"enabled": ["a"]}}) + assert replaced is not immutable + assert immutable.resolve_situation("one").disabled == ["a"] + with pytest.raises(ValueError, match='Unknown situation: "one"'): + replaced.resolve_situation("one") + + mutable = create_tool_set([a], mutable=True) + assert mutable.define_situations({"one": {"disabled": ["a"]}}) is mutable + assert mutable.resolve_situation("one").disabled == ["a"] + mutable.define_situations({"two": {"enabled": ["a"]}}) + with pytest.raises(ValueError, match='Unknown situation: "one"'): + mutable.resolve_situation("one") + + +class TestSharedContext: + def test_predicate_context_drives_activation(self) -> None: + def is_authenticated(inp: ActivationInput) -> bool: + context = inp.get("context") + if not context: + return False + return bool(context["isAuthenticated"]) + + ts = create_tool_set([a]).activate_when("a", is_authenticated) + assert ts.resolve({"context": {"isAuthenticated": True, "userId": "u1"}}).active_tools == ["a"] + assert ts.resolve({"context": {"isAuthenticated": False, "userId": "u1"}}).active_tools == [] + + def test_untyped_context_default(self) -> None: + ts = create_tool_set([a]).activate_when("a", lambda i: ctx(i).get("enabled") is True) + assert ts.resolve({"context": {"enabled": True}}).active_tools == ["a"] + + +class TestCallModelSpreadShape: + def test_produces_tools_and_active_tools_suitable_for_call_model_spread(self) -> None: + snapshot = create_tool_set([a, b, c]).deactivate("b").resolve() + for_call_model = snapshot.call_model + assert for_call_model == {"tools": [a, c], "active_tools": ["a", "c"]} + # Exactly these two keys: snapshot metadata never leaks into the request. + assert list(for_call_model) == ["tools", "active_tools"] + + def test_tool_set_create_classmethod_matches_factory(self) -> None: + ts = ToolSet.create(tools=[a, b], mutable=True) + assert ts.mutable is True + assert ts.deactivate("a") is ts + assert ts.resolve().active_tools == ["b"] + + +class TestArgumentValidation: + def test_activate_when_requires_a_predicate_with_a_name(self) -> None: + with pytest.raises(ValueError, match="requires a predicate"): + create_tool_set([a]).activate_when("a") + + def test_activate_when_rejects_non_mapping_non_string(self) -> None: + with pytest.raises(ValueError, match="requires a name\\+predicate or predicate map"): + create_tool_set([a]).deactivate_when(["a"]) # type: ignore[arg-type] + + def test_predicate_map_skips_non_callable_entries(self) -> None: + resolved = create_tool_set([a, b]).activate_when({"a": None, "b": lambda _i: False}).resolve() + assert resolved.active_tools == ["a"] + assert resolved.status_by_tool["a"] == {"enabled": True, "reason": "default"} diff --git a/tests/unit/test_tool_terminal_empty_final.py b/tests/unit/test_tool_terminal_empty_final.py index f5e75cb..0558e27 100644 --- a/tests/unit/test_tool_terminal_empty_final.py +++ b/tests/unit/test_tool_terminal_empty_final.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import Any, Dict from openrouter_agent import call_model, tool @@ -139,9 +140,16 @@ async def test_throws_on_empty_final_when_strict_final_response_is_true() -> Non ).get_text() except ValueError as error: raised = True - assert "Invalid final response: empty or invalid output" in str(error) + # Upstream delta 680bceb..50a1ec2 tightened this from a substring match + # to an anchored prefix naming the empty array and the response id. + assert re.match( + r"^Invalid final response: empty or invalid output \u2014 output array is empty \(length 0\) " + r'for response "resp_empty"', + str(error), + ) assert raised + # No retry when strict. assert len(client.requests) == 2 @@ -151,8 +159,13 @@ async def test_still_throws_on_empty_output_when_no_tool_rounds_completed() -> N raised = False try: await call_model(client, {"model": "test-model", "input": "hello"}).get_text() - except ValueError: + except ValueError as error: raised = True + assert re.match( + r"^Invalid final response: empty or invalid output \u2014 output array is empty \(length 0\) " + r'for response "resp_empty"\. The model returned no output items', + str(error), + ) assert raised assert len(client.requests) == 1 diff --git a/tests/unit/test_turn_context.py b/tests/unit/test_turn_context.py new file mode 100644 index 0000000..e453625 --- /dev/null +++ b/tests/unit/test_turn_context.py @@ -0,0 +1,96 @@ +"""Ports `packages/agent/tests/unit/turn-context.test.ts` plus the +`is_input_item` / `output_to_input_items` helpers added to turn-context.ts. + +Python's turn context uses snake_case keys (`number_of_turns`, `tool_call`, +`turn_request`) for upstream's `numberOfTurns` / `toolCall` / `turnRequest`. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +from openrouter_agent.turn_context import ( + build_turn_context, + is_input_item, + normalize_input_to_array, + output_to_input_items, +) + + +def test_build_turn_context_minimal() -> None: + context = build_turn_context(number_of_turns=0) + + assert context == {"number_of_turns": 0} + assert "tool_call" not in context + assert "turn_request" not in context + + +def test_build_turn_context_includes_tool_call() -> None: + tool_call = {"type": "function_call", "callId": "call_1", "name": "search", "arguments": "{}", "id": "call_1"} + + context = build_turn_context(number_of_turns=2, tool_call=tool_call) + + assert context["number_of_turns"] == 2 + assert context["tool_call"] is tool_call + assert "turn_request" not in context + + +def test_build_turn_context_includes_turn_request() -> None: + turn_request: Dict[str, Any] = {"model": "openai/gpt-4", "input": []} + + context = build_turn_context(number_of_turns=1, turn_request=turn_request) + + assert context["turn_request"] is turn_request + + +def test_build_turn_context_includes_both() -> None: + tool_call = {"type": "function_call", "callId": "call_2", "name": "browse", "arguments": "{}", "id": "call_2"} + turn_request = {"model": "openai/gpt-4"} + + context = build_turn_context(number_of_turns=3, tool_call=tool_call, turn_request=turn_request) + + assert context == {"number_of_turns": 3, "tool_call": tool_call, "turn_request": turn_request} + + +def test_normalize_input_converts_string_to_single_user_message() -> None: + assert normalize_input_to_array("Hello!") == [{"role": "user", "content": "Hello!"}] + + +def test_normalize_input_passes_list_through_by_reference() -> None: + items: List[Any] = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}] + + assert normalize_input_to_array(items) is items + + +def test_normalize_input_handles_empty_string() -> None: + assert normalize_input_to_array("") == [{"role": "user", "content": ""}] + + +def test_is_input_item_accepts_typed_or_role_bearing_objects() -> None: + assert is_input_item({"type": "function_call"}) is True + assert is_input_item({"role": "user", "content": "x"}) is True + + +def test_is_input_item_rejects_non_items() -> None: + assert is_input_item(None) is False + assert is_input_item("message") is False + assert is_input_item(42) is False + assert is_input_item([{"type": "message"}]) is False + assert is_input_item({}) is False + assert is_input_item({"type": 1, "role": None}) is False + + +def test_is_input_item_reads_attributes_of_model_objects() -> None: + class OutputItem: + type = "message" + + assert is_input_item(OutputItem()) is True + + +def test_output_to_input_items_filters_and_wraps() -> None: + message = {"type": "message", "role": "assistant", "content": []} + call = {"type": "function_call", "callId": "c1", "name": "x", "arguments": "{}"} + + assert output_to_input_items([message, None, "junk", call]) == [message, call] + assert output_to_input_items(message) == [message] + assert output_to_input_items(None) == [] diff --git a/tests/unit/test_turn_end_race_condition.py b/tests/unit/test_turn_end_race_condition.py index 627264d..fd8bae9 100644 --- a/tests/unit/test_turn_end_race_condition.py +++ b/tests/unit/test_turn_end_race_condition.py @@ -50,7 +50,7 @@ async def _gated_stream(events: List[Dict[str, Any]], gate: asyncio.Event) -> As async def test_broadcaster_silently_drops_events_pushed_after_complete() -> None: """Characterization: this is the mechanism behind the dropped turn.end.""" - broadcaster = ToolEventBroadcaster() + broadcaster: ToolEventBroadcaster[Any] = ToolEventBroadcaster() consumer = broadcaster.create_consumer() broadcaster.push({"type": "turn.start"}) @@ -66,7 +66,7 @@ async def test_broadcaster_silently_drops_events_pushed_after_complete() -> None async def test_buggy_pattern_drops_turn_end_when_complete_precedes_pipe() -> None: """Fire-and-forget pipe + early complete() loses turn.end. Documents the bug.""" - broadcaster = ToolEventBroadcaster() + broadcaster: ToolEventBroadcaster[Any] = ToolEventBroadcaster() gate = asyncio.Event() stream = ReusableReadableStream( _gated_stream([{"type": "response.output_text.delta"}, {"type": "response.completed"}], gate) @@ -102,7 +102,7 @@ async def pipe() -> None: async def test_fixed_pattern_preserves_turn_end_when_pipe_is_awaited() -> None: """Awaiting the pipe before complete() keeps turn.end, ordered, with turnNumber.""" - broadcaster = ToolEventBroadcaster() + broadcaster: ToolEventBroadcaster[Any] = ToolEventBroadcaster() gate = asyncio.Event() stream = ReusableReadableStream( _gated_stream([{"type": "response.output_text.delta"}, {"type": "response.completed"}], gate) diff --git a/tests/unit/test_user_agent.py b/tests/unit/test_user_agent.py index b14c8f7..0570b41 100644 --- a/tests/unit/test_user_agent.py +++ b/tests/unit/test_user_agent.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.metadata +import re import httpx @@ -8,6 +9,34 @@ from openrouter.errors.responsevalidationerror import ResponseValidationError from openrouter_agent import OpenRouter, call_model from openrouter_agent.openrouter import _agent_user_agent_token, _append_agent_user_agent +from openrouter_agent.user_agent import AGENT_USER_AGENT, PACKAGE_VERSION, build_agent_user_agent + + +def test_agent_user_agent_composes_sdk_and_agent_package_versions() -> None: + """Ports tests/unit/user-agent.test.ts. + + Upstream asserts ` @openrouter/agent/`; this port appends the PyPI + distribution name instead (contract, Idiomatic Divergences #7). Do not + "fix" this to the TypeScript literal. + """ + assert PACKAGE_VERSION == importlib.metadata.version("openrouter-agent-sdk") + assert AGENT_USER_AGENT == f"{__user_agent__} openrouter-agent-sdk/{PACKAGE_VERSION}" + assert AGENT_USER_AGENT.startswith("speakeasy-sdk/python ") + assert re.search(r" openrouter-agent-sdk/\d+\.\d+\.\d+$", AGENT_USER_AGENT) + assert "@openrouter/agent" not in AGENT_USER_AGENT + + +def test_default_client_user_agent_equals_agent_user_agent() -> None: + # Ports openrouter.test.ts "should default to the Agent SDK user agent". + assert OpenRouter(api_key="test").sdk_configuration.user_agent == AGENT_USER_AGENT + + +def test_build_agent_user_agent_falls_back_to_bare_name_without_metadata() -> None: + def missing(_name: str) -> str: + raise importlib.metadata.PackageNotFoundError + + assert build_agent_user_agent(missing) == f"{__user_agent__} openrouter-agent-sdk" + assert build_agent_user_agent(lambda _name: "9.9.9") == f"{__user_agent__} openrouter-agent-sdk/9.9.9" def test_constructed_client_appends_agent_user_agent() -> None: diff --git a/tests/vectors/doom-loop-fingerprints.json b/tests/vectors/doom-loop-fingerprints.json new file mode 100644 index 0000000..db2c5e5 --- /dev/null +++ b/tests/vectors/doom-loop-fingerprints.json @@ -0,0 +1,132 @@ +{ + "description": "Cross-port doom-loop fingerprint conformance vectors. fingerprint = sha256(utf8(toolName + \"\\n\" + jcs(keyMaterial))) for tool calls, sha256(utf8(jcs(keyMaterial))) for bare key material. jcs = RFC 8785 canonical JSON. Ports MUST use an RFC 8785 implementation (pip jcs, cyberphone/json-canonicalization), NOT their stdlib JSON serializer. All hex lowercase.", + "toolCallVectors": [ + { + "name": "basic bash identity", + "toolName": "bash", + "keyMaterial": { + "command": "ls -la", + "cwd": "/tmp" + }, + "jcs": "{\"command\":\"ls -la\",\"cwd\":\"/tmp\"}", + "fingerprint": "c3430466c9a1a7c11c8e23328fcf3057189c44d86839527ac6e61bb4574410d4" + }, + { + "name": "key order insensitivity (same fingerprint as basic bash identity)", + "toolName": "bash", + "keyMaterial": { + "cwd": "/tmp", + "command": "ls -la" + }, + "jcs": "{\"command\":\"ls -la\",\"cwd\":\"/tmp\"}", + "fingerprint": "c3430466c9a1a7c11c8e23328fcf3057189c44d86839527ac6e61bb4574410d4" + }, + { + "name": "empty object (repeated empty call)", + "toolName": "list_tasks", + "keyMaterial": {}, + "jcs": "{}", + "fingerprint": "b84a3e324e9dc95335f72cfee5e5898465349be72cb1348fe7927c49c8d65d92" + }, + { + "name": "non-ascii + emoji", + "toolName": "web_search", + "keyMaterial": { + "query": "café 🎉" + }, + "jcs": "{\"query\":\"café 🎉\"}", + "fingerprint": "a8bd7e02e874cd1d68db436c3228a42ed2d656ddd30d3195db0e72378479b160" + }, + { + "name": "negative zero collapses to 0 (JCS)", + "toolName": "calc", + "keyMaterial": { + "value": 0 + }, + "jcs": "{\"value\":0}", + "fingerprint": "2c0292f2f8e9099f2879c189b8bd0d61fdc24dea98f06ec1d96f17f8ccb86f7d" + }, + { + "name": "large magnitude number 1e21 (JCS exponent form)", + "toolName": "calc", + "keyMaterial": { + "value": 1e+21 + }, + "jcs": "{\"value\":1e+21}", + "fingerprint": "53f17cc32c78fee57f8e6aba17c4d5f75ca2aa5a1d06d359880c584993dcaaff" + }, + { + "name": "nested structures with arrays", + "toolName": "query", + "keyMaterial": { + "filter": { + "tags": [ + "b", + "a" + ], + "depth": 2 + }, + "sort": null + }, + "jcs": "{\"filter\":{\"depth\":2,\"tags\":[\"b\",\"a\"]},\"sort\":null}", + "fingerprint": "b42e583c626c3b2bcbd5adf86b7099d784ef15398f2e0c530262372e73850f4e" + }, + { + "name": "lone surrogate escapes as \\ud800 (JSON.stringify)", + "toolName": "echo", + "keyMaterial": { + "s": "\ud800" + }, + "jcs": "{\"s\":\"\\ud800\"}", + "fingerprint": "9d901512fe3c9139d48aeaec555bc91c5b8069950bc17a6d014c85bf03c0d8eb" + }, + { + "name": "unicode normalization NOT applied (NFC vs NFD differ)", + "toolName": "echo", + "keyMaterial": { + "s": "é" + }, + "jcs": "{\"s\":\"é\"}", + "fingerprint": "a281d1b32556677dee1b6039b78cde21229a6777874c6dc8eafc4bff64058568" + }, + { + "name": "string primitive key material", + "toolName": "web_search", + "keyMaterial": "openrouter agent sdk", + "jcs": "\"openrouter agent sdk\"", + "fingerprint": "dd066597b3c639cd2c905159c3ee5bda98c29add4deef7d20f723583c1e7e177" + } + ], + "keyMaterialVectors": [ + { + "name": "plain phrase", + "keyMaterial": "I am stuck.", + "jcs": "\"I am stuck.\"", + "fingerprint": "37c38ab53f4c3570d8baf87392cb035ef8b2ad98ca1c1e083ea84184173911aa" + }, + { + "name": "whitespace-normalized cross-step text", + "keyMaterial": "Retrying the same plan.", + "jcs": "\"Retrying the same plan.\"", + "fingerprint": "764fb75d0e16f75f0265b68d579c571e704b6ce9ea2b5dea784200cb701fe3df" + } + ], + "rejected": [ + { + "name": "bigint", + "reason": "RFC 8785 has no representation; canonicalize throws, engine falls back to full arguments" + }, + { + "name": "NaN / Infinity", + "reason": "non-finite numbers unrepresentable; throws, engine falls back" + }, + { + "name": "circular reference", + "reason": "throws, engine falls back" + }, + { + "name": "nesting > 64 levels", + "reason": "depth cap; throws, engine falls back" + } + ] +} diff --git a/upstreamer-changelog.md b/upstreamer-changelog.md index 29fa619..9e26fa1 100644 --- a/upstreamer-changelog.md +++ b/upstreamer-changelog.md @@ -1,5 +1,64 @@ # openrouter-agent-sdk Changelog +## Unreleased (partial sync toward `@openrouter/agent` 1.2.0, not yet publishable) + +This is partial work toward the upstream 0.9.0–1.2.0 releases. The package version stays at `0.8.0` because the upstream async-tool system (background/deferred tools, the `task` tool, steering, subagent tools, per-tool cancellation/timeouts, tool concurrency) has **not** been ported yet. Do not publish this state. + +### New + +- **Doom-loop detection** (opt-in `doom_loop=True` or a config dict). It flags repeated identical tool calls (including same-tool fan-outs and invalid-JSON repeats), repeated server-tool calls, and repeated text. It responds through a graduated ladder: `observe`, `steer`, `escalate`, `block`, `stop`. + - The new `DoomLoopDetected` hook fires on each detection, and `SessionEnd` gets a `"doom_loop"` reason. + - Tools declare call identity with `loop_key=`. + - Detector state persists in `ConversationState.doom_loop`. + - `ModelResult.get_doom_loop_verdict()` reports a stop. + - The primitives (`DoomLoopMonitor`, `resolve_doom_loop_option`, `fingerprint_tool_call`, …) are exported. Fingerprints match the shared cross-port conformance vectors. +- **Tool sets:** `create_tool_set(...)` with `activate` / `deactivate` / `activate_when` / `deactivate_when`. There is also a new `active_tools=[...]` option on `call_model`, which filters the client tools sent to the model; server tools always pass. A filtered-to-empty tool list now omits `tools` from the request entirely. +- **`await result.get_usage()`:** aggregate token/cost totals across every model call in a run, with the same shape as `SessionEnd.total_usage`. + - It never raises. + - Usage now accrues even when no hooks are configured. +- **`next_turn_params` can set `tool_choice`**, e.g. to widen an `allowed_tools` set without changing `tools`. + - Context functions now see upstream's defaults for absent fields. + - Invalid keys are warned about and skipped. + - Non-callable values are ignored. + - Non-object call arguments raise a descriptive error. +- **New tool options:** + - `tool(..., strict=True)` is serialized onto the wire definition. + - `tool(..., wire_input_schema={...})` sends a caller-owned JSON Schema while `input_schema` keeps validating. + - `server_tool(config, {"id": ...})` sets a stable id (default `server:`). + - `mark_mcp(tool, {"loop_key", "wire_input_schema"})`. +- **The user agent now includes the package version:** `openrouter-agent-sdk/`. +- **`ReusableReadableStream` and `ToolEventBroadcaster`** support replay compaction (`stream_replay="active-consumers"`) and release their backlog when consumers leave or the stream is cancelled. +- **`from_chat_messages`** now emits `function_call` items for assistant tool calls, which were previously dropped. + +### Behavior changes + +- **Stop conditions now count executed tool rounds, as upstream does,** not every model response. `step_count_is(n)` therefore allows one more tool round than before. Use `step_count_is(0)` to stop on the first response's tool calls. +- **A forced `tool_choice` relaxes to `auto` once it has produced a tool call.** This covers `"required"`, a specific tool, and `allowed_tools` with `mode: "required"`. The relaxation persists across approval/HITL/client-tool resumes, so the model can answer in text instead of being forced to call tools until the step budget runs out. Callable `tool_choice` (and other callable request params) are now re-resolved every turn. +- **Approval hardening:** + - **Duplicate IDs:** a call ID listed twice in `approve_tool_calls` / `reject_tool_calls` no longer executes twice. Passing the same ID in both lists raises before anything runs. + - **Resuming with no decision:** resuming an `awaiting_approval` conversation with no decision stays paused instead of sending a new request. The input passed with that call is not sent or saved. An `awaiting_hitl` conversation still resumes with a `function_call_output` for a paused call. + - **Partial decisions:** the remaining calls stay pending. + - **Getter errors while paused:** `get_response()`, `get_text()` and the stream getters raise an explicit "Conversation is awaiting approval ..." error that lists pending call IDs. `requires_approval()`, `get_pending_tool_calls()` and `get_state()` keep working. + - **Auto-approved sibling calls:** when a turn pauses for approval, its auto-approved sibling calls now run before the pause and are replayed on resume. Previously they were left without outputs. If `stop_when` is already met and `allow_final_response=False`, they are recorded as skipped instead of executed. + - **`allow_final_response` path:** this path now enforces the approval gate. Each call is gated at most once per run. + - **`require_approval` predicates:** they receive schema-parsed arguments (defaults applied), and schema-invalid arguments fail closed. + - **`PreToolUse` timing:** `PreToolUse` now runs before execution, as part of the approval gate. A mutated input is validated and re-gated, and a pending call records `pre_tool_use_applied=True` so the hook is not applied twice on resume. + - **Rejection output:** a rejection now produces `{"error": "Rejected by user"}`. +- **Truncated responses:** a response truncated at `max_output_tokens` never executes its tool calls, including calls completed before the cut-off. The run finalizes with the incomplete response. +- **Generator tools** no longer accumulate a `preliminary_results` list on the result or on `tool.result` events. Yields are still delivered live. +- **Tool events:** `tool.result` and `tool.preliminary_result` events, and the `get_tool_stream()` projection, now carry `toolName` / `tool_name`. +- **Execute context:** the execute context's `tool_call` is now always populated with a wire-shaped `function_call` item, including on the streaming loop and on direct executor calls. +- **Streams:** a stream that never closes after its terminal `response.completed` event no longer hangs the run. +- **`is_claude_style_messages`** now requires a Claude-specific content block, as upstream does. +- **`is_stop_condition_met`** evaluates every condition and stops only on an exact `True`. +- **Final-response error messages:** `Invalid final response` errors now name the defect (empty array vs. wrong type, and which fields are missing). The existing message prefixes are unchanged. + +### Compatibility notes + +- **Predicate arguments:** for Pydantic `input_schema`s, approval predicates receive `model_dump()`ed dicts, because the predicate contract is a mapping of arguments. +- **`strict` and `description`:** when unset they are omitted from the wire definition, where upstream sends `null`. +- **MCP:** the MCP integration upstream moved into `@openrouter/agent/mcp` is not part of this package. + ## 0.8.0 — first PyPI release First published release. The version is `0.8.0`, not `0.0.1`, because this