Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/rius/langchain_span_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
#: The instrumentor releases whose tracer internals were read and found as
#: this module expects. A later release may have reshaped them; it is left
#: alone.
VERIFIED_VERSIONS = ("0.1.67", "0.1.86")
VERIFIED_VERSIONS = ("0.1.67", "0.1.87")

#: The instrumentor's tracer the hooks look spans up in. Set by install().
_tracer: Any = None
Expand Down
2 changes: 2 additions & 0 deletions src/rius/llama_index_span_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@

#: The instrumentor releases whose span handling was read and found as this
#: module expects. A later release may have reshaped it; it is left alone.
#: From 4.7.7 the instrumentor runs each step of a streamed call with its span
#: current by itself, so the range ends at 4.7.6 for good.
VERIFIED_VERSIONS = ("4.4.3", "4.7.6")

#: The methods whose result is a stream that makes the provider request later.
Expand Down
14 changes: 13 additions & 1 deletion src/rius/normalization.py
Original file line number Diff line number Diff line change
Expand Up @@ -575,6 +575,11 @@ def error_type_from_exception_event(span: ReadableSpan) -> dict[str, Any]:
#: keys, not one key), so it is spelled here.
_LLM_TOOL_SCHEMA = re.compile(r"llm\.tools\.(\d+)\.tool\.json_schema")

#: The members of a tool the anthropic instrumentor (3.3.6 on) also writes on
#: their own beside its schema, as ``llm.tools.{i}.tool.<member>``. Copies of
#: the schema's own fields, so they go wherever the schema goes.
_LLM_TOOL_SCHEMA_COPIES = ("name", "description")


def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str, Any] | None:
"""Reassemble a span's tool definitions into one ``gen_ai.tool.definitions``.
Expand All @@ -594,7 +599,9 @@ def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str
parsed and re-serialized as one array, in index order, in the
conventions' shape, as on the native ``set_tool_definitions`` (see
``_tool_definitions``; a built-in tool's type is prefixed with the
span's ``gen_ai.provider.name``). The indexed keys are then deleted.
span's ``gen_ai.provider.name``). The indexed keys are then deleted,
with the ``llm.tools.N.tool.name`` and ``.description`` copies some
instrumentors write beside a schema.
If any one schema does not parse, the family is left untouched rather
than reassembled without it: it is content by prefix already, so
nothing escapes, and a partial array would be a silent loss.
Expand Down Expand Up @@ -634,6 +641,11 @@ def normalize_tool_definitions(attributes: Mapping[str, Any] | None) -> dict[str
return None
schemas.append(schema)
drop = set(indexed.values())
drop.update(
f"llm.tools.{index}.tool.{member}"
for index in indexed
for member in _LLM_TOOL_SCHEMA_COPIES
)
rebuilt = {k: v for k, v in attributes.items() if k not in drop}
# litellm and langchain write the same definitions into the request
# bag as well. They are duplicates of the ones just reassembled, and
Expand Down
3 changes: 2 additions & 1 deletion tests/test_langchain_span_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -501,7 +501,8 @@ def test_uninstall_restores_the_hooked_classes(openai_answers: None) -> None:
("0.1.81", True),
("0.1.85", True),
("0.1.86", True),
("0.1.87", False),
("0.1.87", True),
("0.1.88", False),
("0.2.0", False),
],
)
Expand Down
20 changes: 19 additions & 1 deletion tests/test_llama_index_span_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import threading
from collections.abc import AsyncIterator, Iterator
from concurrent.futures import ThreadPoolExecutor
from importlib.metadata import version
from typing import Any

import pytest
Expand All @@ -42,6 +43,15 @@
LlamaIndexInstrumentor = llama_index_instrumentation.LlamaIndexInstrumentor
OpenAIInstrumentor = openai_instrumentation.OpenAIInstrumentor


#: The stopgap installs only on the verified releases. From 4.7.7 the
#: instrumentor nests streamed calls by itself and the stopgap stays out, so
#: the tests of how it installs and uninstalls have nothing to look at there.
installs = pytest.mark.skipif(
not _verified(version("openinference-instrumentation-llama-index")),
reason="the stopgap stays out of releases past the verified ones (4.7.7 nests by itself)",
)

CHAT = "https://api.openai.com/v1/chat/completions"
USAGE = {"prompt_tokens": 42, "completion_tokens": 9, "total_tokens": 51}
MESSAGES = [ChatMessage(role="user", content="Capital of France?")]
Expand Down Expand Up @@ -502,6 +512,7 @@ def stream_chat(self, messages: Any, **kwargs: Any) -> Any:
assert names[provider.parent.span_id] == "LaterOpenAI.stream_chat"


@installs
def test_the_subclass_hook_still_runs_an_ancestors_hook(
openai_answers: Answers, monkeypatch: pytest.MonkeyPatch
) -> None:
Expand Down Expand Up @@ -558,6 +569,7 @@ def _stream_methods() -> dict[str, Any]:
return {name: OpenAI.__dict__[name] for name in llama_index_span_context._STREAM_METHODS}


@installs
def test_reinit_does_not_stack_hooks(openai_answers: Answers) -> None:
for _ in range(2):
client = init(
Expand Down Expand Up @@ -587,6 +599,7 @@ def test_reinit_does_not_stack_hooks(openai_answers: Answers) -> None:
assert len(ours) == 1


@installs
def test_installing_twice_does_not_stack(openai_answers: Answers) -> None:
instrumentor = LlamaIndexInstrumentor()
instrumentor.instrument()
Expand All @@ -601,6 +614,7 @@ def test_installing_twice_does_not_stack(openai_answers: Answers) -> None:
instrumentor.uninstrument()


@installs
def test_our_handler_runs_after_the_instrumentors_after_a_reinstrument(
openai_answers: Answers,
) -> None:
Expand All @@ -622,6 +636,7 @@ def test_our_handler_runs_after_the_instrumentors_after_a_reinstrument(
instrumentor.uninstrument()


@installs
def test_uninstall_restores_everything(openai_answers: Answers) -> None:
originals = _stream_methods()
hook = BaseLLM.__dict__.get("__pydantic_init_subclass__")
Expand Down Expand Up @@ -669,11 +684,13 @@ def test_an_unverified_version_installs_nothing(
openai_answers: Answers, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(llama_index_span_context, "version", lambda name: "9.9.9")
originals = _stream_methods()
handlers = list(get_dispatcher().span_handlers)
instrumentor = LlamaIndexInstrumentor()
instrumentor.instrument()
try:
# Taken after instrument(): from 4.7.7 the instrumentor wraps the
# stream methods itself.
originals = _stream_methods()
assert not llama_index_span_context.install(instrumentor)
assert _stream_methods() == originals
assert [h for h in get_dispatcher().span_handlers if h not in handlers] == [
Expand Down Expand Up @@ -818,6 +835,7 @@ def stream() -> Iterator[int]:
assert handler.take_stream_span(result) is None


@installs
def test_uninstall_keeps_a_busy_handler_until_its_call_ends(
openai_answers: Answers,
) -> None:
Expand Down
33 changes: 33 additions & 0 deletions tests/test_normalization_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,39 @@ def test_an_unparseable_schema_leaves_the_whole_family_alone() -> None:
assert normalize_tool_definitions(attributes) is None


def test_a_tools_name_and_description_go_with_its_schema() -> None:
# The anthropic instrumentor (3.3.6 on) writes each tool's name and
# description beside its schema. They are copies of the schema's own
# fields, so they leave with it rather than ride next to the definitions.
attributes = {
**_indexed(_ANTHROPIC_TOOLS),
"llm.tools.0.tool.name": "get_weather",
"llm.tools.0.tool.description": "Weather for a city",
}
out = normalize_tool_definitions(attributes)
assert out is not None
assert _definitions(out) == _ANTHROPIC_SEMCONV
assert not [key for key in out if key.startswith("llm.tools")]


def test_a_name_without_a_schema_is_not_touched() -> None:
# Only the copies of a reassembled schema go: a name at an index with no
# schema is nothing the definitions hold.
attributes = {**_indexed([{"name": "a"}]), "llm.tools.1.tool.name": "b"}
out = normalize_tool_definitions(attributes)
assert out is not None
assert out["llm.tools.1.tool.name"] == "b"


def test_an_unparseable_schema_leaves_the_name_and_description_too() -> None:
attributes = {
"llm.tools.0.tool.json_schema": "{not json",
"llm.tools.0.tool.name": "ok",
"llm.tools.0.tool.description": "fine",
}
assert normalize_tool_definitions(attributes) is None


def test_other_keys_under_llm_tools_are_not_touched() -> None:
attributes = {**_indexed([{"name": "a"}]), "llm.tools.0.tool.something_else": "x"}
out = normalize_tool_definitions(attributes)
Expand Down
Loading