From de0d804b404c553e7b6e9f867e7becfcd093556c Mon Sep 17 00:00:00 2001 From: Pablo Pardo Garcia Date: Tue, 17 Nov 2026 09:46:21 +0100 Subject: [PATCH] fix: cover langchain 0.1.87, llama-index 4.7.7 and anthropic 3.3.6 in the newest-instrumentor checks OpenInference released langchain 0.1.87, llama-index 4.7.7 and anthropic 3.3.6 on 2026-11-16 at 23:47Z, and the scheduled newest-instrumentors run went red. langchain 0.1.87 only adds a retriever document id. Widened the LangChain stopgap to it. llama-index 4.7.7 runs each step of a streamed LLM call with its span current by itself, so the LlamaIndex stopgap stays out from there and its range keeps ending at 4.7.6. The tests of how it installs skip on releases past it. anthropic 3.3.6 writes each tool's name and description as llm.tools.N.tool.name and .description beside the schema. They are copies of the schema, so they now leave with it when the definitions are reassembled. --- src/rius/langchain_span_context.py | 2 +- src/rius/llama_index_span_context.py | 2 ++ src/rius/normalization.py | 14 ++++++++++- tests/test_langchain_span_context.py | 3 ++- tests/test_llama_index_span_context.py | 20 +++++++++++++++- tests/test_normalization_tools.py | 33 ++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/rius/langchain_span_context.py b/src/rius/langchain_span_context.py index 4b2b803..c607aa4 100644 --- a/src/rius/langchain_span_context.py +++ b/src/rius/langchain_span_context.py @@ -63,7 +63,7 @@ #: The instrumentor releases whose tracer internals were read and found as #: this module expects. A later release may have reshaped them; it is left #: alone. -VERIFIED_VERSIONS = ("0.1.67", "0.1.86") +VERIFIED_VERSIONS = ("0.1.67", "0.1.87") #: The instrumentor's tracer the hooks look spans up in. Set by install(). _tracer: Any = None diff --git a/src/rius/llama_index_span_context.py b/src/rius/llama_index_span_context.py index 7cda030..f9df620 100644 --- a/src/rius/llama_index_span_context.py +++ b/src/rius/llama_index_span_context.py @@ -95,6 +95,8 @@ #: The instrumentor releases whose span handling was read and found as this #: module expects. A later release may have reshaped it; it is left alone. +#: From 4.7.7 the instrumentor runs each step of a streamed call with its span +#: current by itself, so the range ends at 4.7.6 for good. VERIFIED_VERSIONS = ("4.4.3", "4.7.6") #: The methods whose result is a stream that makes the provider request later. diff --git a/src/rius/normalization.py b/src/rius/normalization.py index 2a6d541..a1da18c 100644 --- a/src/rius/normalization.py +++ b/src/rius/normalization.py @@ -575,6 +575,11 @@ def error_type_from_exception_event(span: ReadableSpan) -> dict[str, Any]: #: keys, not one key), so it is spelled here. _LLM_TOOL_SCHEMA = re.compile(r"llm\.tools\.(\d+)\.tool\.json_schema") +#: The members of a tool the anthropic instrumentor (3.3.6 on) also writes on +#: their own beside its schema, as ``llm.tools.{i}.tool.``. Copies of +#: the schema's own fields, so they go wherever the schema goes. +_LLM_TOOL_SCHEMA_COPIES = ("name", "description") + def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str, Any] | None: """Reassemble a span's tool definitions into one ``gen_ai.tool.definitions``. @@ -594,7 +599,9 @@ def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str parsed and re-serialized as one array, in index order, in the conventions' shape, as on the native ``set_tool_definitions`` (see ``_tool_definitions``; a built-in tool's type is prefixed with the - span's ``gen_ai.provider.name``). The indexed keys are then deleted. + span's ``gen_ai.provider.name``). The indexed keys are then deleted, + with the ``llm.tools.N.tool.name`` and ``.description`` copies some + instrumentors write beside a schema. If any one schema does not parse, the family is left untouched rather than reassembled without it: it is content by prefix already, so nothing escapes, and a partial array would be a silent loss. @@ -634,6 +641,11 @@ def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str return None schemas.append(schema) drop = set(indexed.values()) + drop.update( + f"llm.tools.{index}.tool.{member}" + for index in indexed + for member in _LLM_TOOL_SCHEMA_COPIES + ) rebuilt = {k: v for k, v in attributes.items() if k not in drop} # litellm and langchain write the same definitions into the request # bag as well. They are duplicates of the ones just reassembled, and diff --git a/tests/test_langchain_span_context.py b/tests/test_langchain_span_context.py index 220b8c7..69940af 100644 --- a/tests/test_langchain_span_context.py +++ b/tests/test_langchain_span_context.py @@ -501,7 +501,8 @@ def test_uninstall_restores_the_hooked_classes(openai_answers: None) -> None: ("0.1.81", True), ("0.1.85", True), ("0.1.86", True), - ("0.1.87", False), + ("0.1.87", True), + ("0.1.88", False), ("0.2.0", False), ], ) diff --git a/tests/test_llama_index_span_context.py b/tests/test_llama_index_span_context.py index 41555ef..89055e6 100644 --- a/tests/test_llama_index_span_context.py +++ b/tests/test_llama_index_span_context.py @@ -17,6 +17,7 @@ import threading from collections.abc import AsyncIterator, Iterator from concurrent.futures import ThreadPoolExecutor +from importlib.metadata import version from typing import Any import pytest @@ -42,6 +43,15 @@ LlamaIndexInstrumentor = llama_index_instrumentation.LlamaIndexInstrumentor OpenAIInstrumentor = openai_instrumentation.OpenAIInstrumentor + +#: The stopgap installs only on the verified releases. From 4.7.7 the +#: instrumentor nests streamed calls by itself and the stopgap stays out, so +#: the tests of how it installs and uninstalls have nothing to look at there. +installs = pytest.mark.skipif( + not _verified(version("openinference-instrumentation-llama-index")), + reason="the stopgap stays out of releases past the verified ones (4.7.7 nests by itself)", +) + CHAT = "https://api.openai.com/v1/chat/completions" USAGE = {"prompt_tokens": 42, "completion_tokens": 9, "total_tokens": 51} MESSAGES = [ChatMessage(role="user", content="Capital of France?")] @@ -502,6 +512,7 @@ def stream_chat(self, messages: Any, **kwargs: Any) -> Any: assert names[provider.parent.span_id] == "LaterOpenAI.stream_chat" +@installs def test_the_subclass_hook_still_runs_an_ancestors_hook( openai_answers: Answers, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -558,6 +569,7 @@ def _stream_methods() -> dict[str, Any]: return {name: OpenAI.__dict__[name] for name in llama_index_span_context._STREAM_METHODS} +@installs def test_reinit_does_not_stack_hooks(openai_answers: Answers) -> None: for _ in range(2): client = init( @@ -587,6 +599,7 @@ def test_reinit_does_not_stack_hooks(openai_answers: Answers) -> None: assert len(ours) == 1 +@installs def test_installing_twice_does_not_stack(openai_answers: Answers) -> None: instrumentor = LlamaIndexInstrumentor() instrumentor.instrument() @@ -601,6 +614,7 @@ def test_installing_twice_does_not_stack(openai_answers: Answers) -> None: instrumentor.uninstrument() +@installs def test_our_handler_runs_after_the_instrumentors_after_a_reinstrument( openai_answers: Answers, ) -> None: @@ -622,6 +636,7 @@ def test_our_handler_runs_after_the_instrumentors_after_a_reinstrument( instrumentor.uninstrument() +@installs def test_uninstall_restores_everything(openai_answers: Answers) -> None: originals = _stream_methods() hook = BaseLLM.__dict__.get("__pydantic_init_subclass__") @@ -669,11 +684,13 @@ def test_an_unverified_version_installs_nothing( openai_answers: Answers, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(llama_index_span_context, "version", lambda name: "9.9.9") - originals = _stream_methods() handlers = list(get_dispatcher().span_handlers) instrumentor = LlamaIndexInstrumentor() instrumentor.instrument() try: + # Taken after instrument(): from 4.7.7 the instrumentor wraps the + # stream methods itself. + originals = _stream_methods() assert not llama_index_span_context.install(instrumentor) assert _stream_methods() == originals assert [h for h in get_dispatcher().span_handlers if h not in handlers] == [ @@ -818,6 +835,7 @@ def stream() -> Iterator[int]: assert handler.take_stream_span(result) is None +@installs def test_uninstall_keeps_a_busy_handler_until_its_call_ends( openai_answers: Answers, ) -> None: diff --git a/tests/test_normalization_tools.py b/tests/test_normalization_tools.py index c3a0e98..f45f565 100644 --- a/tests/test_normalization_tools.py +++ b/tests/test_normalization_tools.py @@ -194,6 +194,39 @@ def test_an_unparseable_schema_leaves_the_whole_family_alone() -> None: assert normalize_tool_definitions(attributes) is None +def test_a_tools_name_and_description_go_with_its_schema() -> None: + # The anthropic instrumentor (3.3.6 on) writes each tool's name and + # description beside its schema. They are copies of the schema's own + # fields, so they leave with it rather than ride next to the definitions. + attributes = { + **_indexed(_ANTHROPIC_TOOLS), + "llm.tools.0.tool.name": "get_weather", + "llm.tools.0.tool.description": "Weather for a city", + } + out = normalize_tool_definitions(attributes) + assert out is not None + assert _definitions(out) == _ANTHROPIC_SEMCONV + assert not [key for key in out if key.startswith("llm.tools")] + + +def test_a_name_without_a_schema_is_not_touched() -> None: + # Only the copies of a reassembled schema go: a name at an index with no + # schema is nothing the definitions hold. + attributes = {**_indexed([{"name": "a"}]), "llm.tools.1.tool.name": "b"} + out = normalize_tool_definitions(attributes) + assert out is not None + assert out["llm.tools.1.tool.name"] == "b" + + +def test_an_unparseable_schema_leaves_the_name_and_description_too() -> None: + attributes = { + "llm.tools.0.tool.json_schema": "{not json", + "llm.tools.0.tool.name": "ok", + "llm.tools.0.tool.description": "fine", + } + assert normalize_tool_definitions(attributes) is None + + def test_other_keys_under_llm_tools_are_not_touched() -> None: attributes = {**_indexed([{"name": "a"}]), "llm.tools.0.tool.something_else": "x"} out = normalize_tool_definitions(attributes)