diff --git a/src/rius/langchain_span_context.py b/src/rius/langchain_span_context.py index 4b2b803..c607aa4 100644 --- a/src/rius/langchain_span_context.py +++ b/src/rius/langchain_span_context.py @@ -63,7 +63,7 @@ #: The instrumentor releases whose tracer internals were read and found as #: this module expects. A later release may have reshaped them; it is left #: alone. -VERIFIED_VERSIONS = ("0.1.67", "0.1.86") +VERIFIED_VERSIONS = ("0.1.67", "0.1.87") #: The instrumentor's tracer the hooks look spans up in. Set by install(). _tracer: Any = None diff --git a/src/rius/llama_index_span_context.py b/src/rius/llama_index_span_context.py index 7cda030..f9df620 100644 --- a/src/rius/llama_index_span_context.py +++ b/src/rius/llama_index_span_context.py @@ -95,6 +95,8 @@ #: The instrumentor releases whose span handling was read and found as this #: module expects. A later release may have reshaped it; it is left alone. +#: From 4.7.7 the instrumentor runs each step of a streamed call with its span +#: current by itself, so the range ends at 4.7.6 for good. VERIFIED_VERSIONS = ("4.4.3", "4.7.6") #: The methods whose result is a stream that makes the provider request later. diff --git a/src/rius/normalization.py b/src/rius/normalization.py index 2a6d541..a1da18c 100644 --- a/src/rius/normalization.py +++ b/src/rius/normalization.py @@ -575,6 +575,11 @@ def error_type_from_exception_event(span: ReadableSpan) -> dict[str, Any]: #: keys, not one key), so it is spelled here. _LLM_TOOL_SCHEMA = re.compile(r"llm\.tools\.(\d+)\.tool\.json_schema") +#: The members of a tool the anthropic instrumentor (3.3.6 on) also writes on +#: their own beside its schema, as ``llm.tools.{i}.tool.``. Copies of +#: the schema's own fields, so they go wherever the schema goes. +_LLM_TOOL_SCHEMA_COPIES = ("name", "description") + def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str, Any] | None: """Reassemble a span's tool definitions into one ``gen_ai.tool.definitions``. @@ -594,7 +599,9 @@ def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str parsed and re-serialized as one array, in index order, in the conventions' shape, as on the native ``set_tool_definitions`` (see ``_tool_definitions``; a built-in tool's type is prefixed with the - span's ``gen_ai.provider.name``). The indexed keys are then deleted. + span's ``gen_ai.provider.name``). The indexed keys are then deleted, + with the ``llm.tools.N.tool.name`` and ``.description`` copies some + instrumentors write beside a schema. If any one schema does not parse, the family is left untouched rather than reassembled without it: it is content by prefix already, so nothing escapes, and a partial array would be a silent loss. @@ -634,6 +641,11 @@ def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str return None schemas.append(schema) drop = set(indexed.values()) + drop.update( + f"llm.tools.{index}.tool.{member}" + for index in indexed + for member in _LLM_TOOL_SCHEMA_COPIES + ) rebuilt = {k: v for k, v in attributes.items() if k not in drop} # litellm and langchain write the same definitions into the request # bag as well. They are duplicates of the ones just reassembled, and diff --git a/tests/test_langchain_span_context.py b/tests/test_langchain_span_context.py index 220b8c7..69940af 100644 --- a/tests/test_langchain_span_context.py +++ b/tests/test_langchain_span_context.py @@ -501,7 +501,8 @@ def test_uninstall_restores_the_hooked_classes(openai_answers: None) -> None: ("0.1.81", True), ("0.1.85", True), ("0.1.86", True), - ("0.1.87", False), + ("0.1.87", True), + ("0.1.88", False), ("0.2.0", False), ], ) diff --git a/tests/test_llama_index_span_context.py b/tests/test_llama_index_span_context.py index 41555ef..89055e6 100644 --- a/tests/test_llama_index_span_context.py +++ b/tests/test_llama_index_span_context.py @@ -17,6 +17,7 @@ import threading from collections.abc import AsyncIterator, Iterator from concurrent.futures import ThreadPoolExecutor +from importlib.metadata import version from typing import Any import pytest @@ -42,6 +43,15 @@ LlamaIndexInstrumentor = llama_index_instrumentation.LlamaIndexInstrumentor OpenAIInstrumentor = openai_instrumentation.OpenAIInstrumentor + +#: The stopgap installs only on the verified releases. From 4.7.7 the +#: instrumentor nests streamed calls by itself and the stopgap stays out, so +#: the tests of how it installs and uninstalls have nothing to look at there. +installs = pytest.mark.skipif( + not _verified(version("openinference-instrumentation-llama-index")), + reason="the stopgap stays out of releases past the verified ones (4.7.7 nests by itself)", +) + CHAT = "https://api.openai.com/v1/chat/completions" USAGE = {"prompt_tokens": 42, "completion_tokens": 9, "total_tokens": 51} MESSAGES = [ChatMessage(role="user", content="Capital of France?")] @@ -502,6 +512,7 @@ def stream_chat(self, messages: Any, **kwargs: Any) -> Any: assert names[provider.parent.span_id] == "LaterOpenAI.stream_chat" +@installs def test_the_subclass_hook_still_runs_an_ancestors_hook( openai_answers: Answers, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -558,6 +569,7 @@ def _stream_methods() -> dict[str, Any]: return {name: OpenAI.__dict__[name] for name in llama_index_span_context._STREAM_METHODS} +@installs def test_reinit_does_not_stack_hooks(openai_answers: Answers) -> None: for _ in range(2): client = init( @@ -587,6 +599,7 @@ def test_reinit_does_not_stack_hooks(openai_answers: Answers) -> None: assert len(ours) == 1 +@installs def test_installing_twice_does_not_stack(openai_answers: Answers) -> None: instrumentor = LlamaIndexInstrumentor() instrumentor.instrument() @@ -601,6 +614,7 @@ def test_installing_twice_does_not_stack(openai_answers: Answers) -> None: instrumentor.uninstrument() +@installs def test_our_handler_runs_after_the_instrumentors_after_a_reinstrument( openai_answers: Answers, ) -> None: @@ -622,6 +636,7 @@ def test_our_handler_runs_after_the_instrumentors_after_a_reinstrument( instrumentor.uninstrument() +@installs def test_uninstall_restores_everything(openai_answers: Answers) -> None: originals = _stream_methods() hook = BaseLLM.__dict__.get("__pydantic_init_subclass__") @@ -669,11 +684,13 @@ def test_an_unverified_version_installs_nothing( openai_answers: Answers, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(llama_index_span_context, "version", lambda name: "9.9.9") - originals = _stream_methods() handlers = list(get_dispatcher().span_handlers) instrumentor = LlamaIndexInstrumentor() instrumentor.instrument() try: + # Taken after instrument(): from 4.7.7 the instrumentor wraps the + # stream methods itself. + originals = _stream_methods() assert not llama_index_span_context.install(instrumentor) assert _stream_methods() == originals assert [h for h in get_dispatcher().span_handlers if h not in handlers] == [ @@ -818,6 +835,7 @@ def stream() -> Iterator[int]: assert handler.take_stream_span(result) is None +@installs def test_uninstall_keeps_a_busy_handler_until_its_call_ends( openai_answers: Answers, ) -> None: diff --git a/tests/test_normalization_tools.py b/tests/test_normalization_tools.py index c3a0e98..f45f565 100644 --- a/tests/test_normalization_tools.py +++ b/tests/test_normalization_tools.py @@ -194,6 +194,39 @@ def test_an_unparseable_schema_leaves_the_whole_family_alone() -> None: assert normalize_tool_definitions(attributes) is None +def test_a_tools_name_and_description_go_with_its_schema() -> None: + # The anthropic instrumentor (3.3.6 on) writes each tool's name and + # description beside its schema. They are copies of the schema's own + # fields, so they leave with it rather than ride next to the definitions. + attributes = { + **_indexed(_ANTHROPIC_TOOLS), + "llm.tools.0.tool.name": "get_weather", + "llm.tools.0.tool.description": "Weather for a city", + } + out = normalize_tool_definitions(attributes) + assert out is not None + assert _definitions(out) == _ANTHROPIC_SEMCONV + assert not [key for key in out if key.startswith("llm.tools")] + + +def test_a_name_without_a_schema_is_not_touched() -> None: + # Only the copies of a reassembled schema go: a name at an index with no + # schema is nothing the definitions hold. + attributes = {**_indexed([{"name": "a"}]), "llm.tools.1.tool.name": "b"} + out = normalize_tool_definitions(attributes) + assert out is not None + assert out["llm.tools.1.tool.name"] == "b" + + +def test_an_unparseable_schema_leaves_the_name_and_description_too() -> None: + attributes = { + "llm.tools.0.tool.json_schema": "{not json", + "llm.tools.0.tool.name": "ok", + "llm.tools.0.tool.description": "fine", + } + assert normalize_tool_definitions(attributes) is None + + def test_other_keys_under_llm_tools_are_not_touched() -> None: attributes = {**_indexed([{"name": "a"}]), "llm.tools.0.tool.something_else": "x"} out = normalize_tool_definitions(attributes)