diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 5f2dba9359..202bd00e07 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -8,7 +8,6 @@ import sentry_sdk from sentry_sdk.ai.utils import ( GEN_AI_ALLOWED_MESSAGE_ROLES, - get_start_span_function, normalize_message_roles, set_data_normalized, transform_content_part, @@ -263,7 +262,7 @@ class SentryLangchainCallback(BaseCallbackHandler): """Callback handler that creates Sentry spans.""" def __init__(self, include_prompts: bool) -> None: - self.span_map: "OrderedDict[UUID, Union[sentry_sdk.tracing.Span, StreamedSpan]]" = OrderedDict() + self.span_map: "OrderedDict[UUID, StreamedSpan]" = OrderedDict() self.include_prompts = include_prompts def _handle_error(self, run_id: "UUID", error: "Any") -> None: @@ -299,12 +298,10 @@ def _create_span( op: str, name: str, origin: str, - ) -> "Union[sentry_sdk.tracing.Span, StreamedSpan]": + ) -> "StreamedSpan": span = None if parent_id: - parent_span: "Optional[Union[sentry_sdk.tracing.Span, StreamedSpan]]" = ( - self.span_map.get(parent_id) - ) + parent_span: "Optional[StreamedSpan]" = self.span_map.get(parent_id) if parent_span: span = ( sentry_sdk.traces.start_span( @@ -320,17 +317,12 @@ def _create_span( ) if span is None: - span_streaming = has_span_streaming_enabled(sentry_sdk.get_client().options) - span = ( - sentry_sdk.traces.start_span( - name=name, - attributes={ - "sentry.op": op, - "sentry.origin": origin, - }, - ) - if span_streaming - else sentry_sdk.start_span(op=op, name=name, origin=origin) + span = sentry_sdk.traces.start_span( + name=name, + attributes={ + "sentry.op": op, + "sentry.origin": origin, + }, ) span.__enter__() @@ -339,7 +331,7 @@ def _create_span( def _exit_span( self: "SentryLangchainCallback", - span: "Union[sentry_sdk.tracing.Span, StreamedSpan]", + span: "StreamedSpan", run_id: "UUID", ) -> None: span.__exit__(None, None, None) @@ -1157,89 +1149,46 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": record_inputs = True record_outputs = True - if has_span_streaming_enabled(client.options): - with sentry_sdk.traces.start_span( - name=f"invoke_agent {run_name}" if run_name else "invoke_agent", - attributes={ - "sentry.op": OP.GEN_AI_INVOKE_AGENT, - "sentry.origin": LangchainIntegration.origin, - SPANDATA.GEN_AI_OPERATION_NAME: "invoke_agent", - SPANDATA.GEN_AI_RESPONSE_STREAMING: False, - }, - ) as span: - if run_name: - span.set_attribute(SPANDATA.GEN_AI_FUNCTION_ID, run_name) - - _set_tools_on_span(span, tools) - - # Run the agent - result = f(self, *args, **kwargs) - - input = result.get("input") - if input is not None and record_inputs: - normalized_messages = normalize_message_roles([input]) - - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, - unpack=False, - ) - - output = result.get("output") - if output is not None and record_outputs: - set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, output) - - return result - else: - start_span_function = get_start_span_function() - - with start_span_function( - op=OP.GEN_AI_INVOKE_AGENT, - name=f"invoke_agent {run_name}" if run_name else "invoke_agent", - origin=LangchainIntegration.origin, - ) as span: - if run_name: - span.set_data(SPANDATA.GEN_AI_FUNCTION_ID, run_name) - - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) + with sentry_sdk.traces.start_span( + name=f"invoke_agent {run_name}" if run_name else "invoke_agent", + attributes={ + "sentry.op": OP.GEN_AI_INVOKE_AGENT, + "sentry.origin": LangchainIntegration.origin, + SPANDATA.GEN_AI_OPERATION_NAME: "invoke_agent", + SPANDATA.GEN_AI_RESPONSE_STREAMING: False, + }, + ) as span: + if run_name: + span.set_attribute(SPANDATA.GEN_AI_FUNCTION_ID, run_name) - _set_tools_on_span(span, tools) + _set_tools_on_span(span, tools) - # Run the agent - result = f(self, *args, **kwargs) + # Run the agent + result = f(self, *args, **kwargs) - input = result.get("input") - if input is not None and record_inputs: - normalized_messages = normalize_message_roles([input]) + input = result.get("input") + if input is not None and record_inputs: + normalized_messages = normalize_message_roles([input]) - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages + scope = sentry_sdk.get_current_scope() + messages_data = ( + truncate_and_annotate_messages(normalized_messages, span, scope) + if not has_span_streaming_enabled(client.options) + else normalized_messages + ) + if messages_data is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + messages_data, + unpack=False, ) - if messages_data is not None: - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, - unpack=False, - ) - output = result.get("output") - if output is not None and record_outputs: - set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, output) + output = result.get("output") + if output is not None and record_outputs: + set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, output) - return result + return result return new_invoke @@ -1264,34 +1213,18 @@ def new_stream(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": record_inputs = True record_outputs = True - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"invoke_agent {run_name}" if run_name else "invoke_agent", - attributes={ - "sentry.op": OP.GEN_AI_INVOKE_AGENT, - "sentry.origin": LangchainIntegration.origin, - SPANDATA.GEN_AI_OPERATION_NAME: "invoke_agent", - SPANDATA.GEN_AI_RESPONSE_STREAMING: True, - }, - ) - - if run_name: - span.set_attribute(SPANDATA.GEN_AI_FUNCTION_ID, run_name) - else: - start_span_function = get_start_span_function() - - span = start_span_function( - op=OP.GEN_AI_INVOKE_AGENT, - name=f"invoke_agent {run_name}" if run_name else "invoke_agent", - origin=LangchainIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) + span = sentry_sdk.traces.start_span( + name=f"invoke_agent {run_name}" if run_name else "invoke_agent", + attributes={ + "sentry.op": OP.GEN_AI_INVOKE_AGENT, + "sentry.origin": LangchainIntegration.origin, + SPANDATA.GEN_AI_OPERATION_NAME: "invoke_agent", + SPANDATA.GEN_AI_RESPONSE_STREAMING: True, + }, + ) - if run_name: - span.set_data(SPANDATA.GEN_AI_FUNCTION_ID, run_name) + if run_name: + span.set_attribute(SPANDATA.GEN_AI_FUNCTION_ID, run_name) _set_tools_on_span(span, tools) @@ -1410,48 +1343,27 @@ def new_embedding_method(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": # TODO: Remove this branch once `send_default_pii` is deprecated record_inputs = True - if has_span_streaming_enabled(client.options): - with sentry_sdk.traces.start_span( - name=f"embeddings {model_name}" if model_name else "embeddings", - attributes={ - "sentry.op": OP.GEN_AI_EMBEDDINGS, - "sentry.origin": LangchainIntegration.origin, - SPANDATA.GEN_AI_OPERATION_NAME: "embeddings", - }, - ) as span: - if model_name: - span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model_name) - - if record_inputs and len(args) > 0: - input_data = args[0] - # Normalize to list format - texts = input_data if isinstance(input_data, list) else [input_data] - set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, texts, unpack=False - ) - - result = f(self, *args, **kwargs) - return result - else: - with sentry_sdk.start_span( - op=OP.GEN_AI_EMBEDDINGS, - name=f"embeddings {model_name}" if model_name else "embeddings", - origin=LangchainIntegration.origin, - ) as span: - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") - if model_name: - span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, model_name) - - if record_inputs and len(args) > 0: - input_data = args[0] - # Normalize to list format - texts = input_data if isinstance(input_data, list) else [input_data] - set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, texts, unpack=False - ) + with sentry_sdk.traces.start_span( + name=f"embeddings {model_name}" if model_name else "embeddings", + attributes={ + "sentry.op": OP.GEN_AI_EMBEDDINGS, + "sentry.origin": LangchainIntegration.origin, + SPANDATA.GEN_AI_OPERATION_NAME: "embeddings", + }, + ) as span: + if model_name: + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model_name) + + if record_inputs and len(args) > 0: + input_data = args[0] + # Normalize to list format + texts = input_data if isinstance(input_data, list) else [input_data] + set_data_normalized( + span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, texts, unpack=False + ) - result = f(self, *args, **kwargs) - return result + result = f(self, *args, **kwargs) + return result return new_embedding_method @@ -1477,47 +1389,26 @@ async def new_async_embedding_method( # TODO: Remove this branch once `send_default_pii` is deprecated record_inputs = True - if has_span_streaming_enabled(client.options): - with sentry_sdk.traces.start_span( - name=f"embeddings {model_name}" if model_name else "embeddings", - attributes={ - "sentry.op": OP.GEN_AI_EMBEDDINGS, - "sentry.origin": LangchainIntegration.origin, - SPANDATA.GEN_AI_OPERATION_NAME: "embeddings", - }, - ) as span: - if model_name: - span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model_name) - - if record_inputs and len(args) > 0: - input_data = args[0] - # Normalize to list format - texts = input_data if isinstance(input_data, list) else [input_data] - set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, texts, unpack=False - ) - - result = await f(self, *args, **kwargs) - return result - else: - with sentry_sdk.start_span( - op=OP.GEN_AI_EMBEDDINGS, - name=f"embeddings {model_name}" if model_name else "embeddings", - origin=LangchainIntegration.origin, - ) as span: - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") - if model_name: - span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, model_name) - - if record_inputs and len(args) > 0: - input_data = args[0] - # Normalize to list format - texts = input_data if isinstance(input_data, list) else [input_data] - set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, texts, unpack=False - ) + with sentry_sdk.traces.start_span( + name=f"embeddings {model_name}" if model_name else "embeddings", + attributes={ + "sentry.op": OP.GEN_AI_EMBEDDINGS, + "sentry.origin": LangchainIntegration.origin, + SPANDATA.GEN_AI_OPERATION_NAME: "embeddings", + }, + ) as span: + if model_name: + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model_name) + + if record_inputs and len(args) > 0: + input_data = args[0] + # Normalize to list format + texts = input_data if isinstance(input_data, list) else [input_data] + set_data_normalized( + span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, texts, unpack=False + ) - result = await f(self, *args, **kwargs) - return result + result = await f(self, *args, **kwargs) + return result return new_async_embedding_method diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 33bc382827..55d269ec3c 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -29,7 +29,6 @@ google = None import sentry_sdk -from sentry_sdk import start_transaction from sentry_sdk.integrations.langchain import ( LangchainIntegration, SentryLangchainCallback, @@ -283,13 +282,10 @@ def get_word_length(word: str) -> int: return len(word) -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_text_completion( sentry_init, - capture_events, capture_items, get_model_response, - span_streaming, ): sentry_init( integrations=[ @@ -300,8 +296,7 @@ def test_langchain_text_completion( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) model_response = get_model_response( @@ -332,82 +327,44 @@ def test_langchain_text_completion( max_tokens=100, openai_api_key="badkey", ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - model.client._client._client, - "send", - return_value=model_response, - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - input_text = "What is the capital of France?" - model.invoke(input_text, config={"run_name": "my-snazzy-pipeline"}) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - llm_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.text_completion" - ] - assert len(llm_spans) > 0 + with patch.object( + model.client._client._client, + "send", + return_value=model_response, + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + input_text = "What is the capital of France?" + model.invoke(input_text, config={"run_name": "my-snazzy-pipeline"}) - llm_span = llm_spans[0] - assert llm_span["name"] == "text_completion gpt-3.5-turbo" - assert llm_span["attributes"]["gen_ai.system"] == "openai" - assert llm_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" - assert llm_span["attributes"]["gen_ai.request.model"] == "gpt-3.5-turbo" - assert ( - llm_span["attributes"]["gen_ai.response.text"] - == "The capital of France is Paris." - ) - assert llm_span["attributes"]["gen_ai.usage.total_tokens"] == 25 - assert llm_span["attributes"]["gen_ai.usage.input_tokens"] == 10 - assert llm_span["attributes"]["gen_ai.usage.output_tokens"] == 15 - else: - events = capture_events() - - with patch.object( - model.client._client._client, - "send", - return_value=model_response, - ) as _, start_transaction(): - input_text = "What is the capital of France?" - model.invoke(input_text, config={"run_name": "my-snazzy-pipeline"}) - - tx = events[0] - assert tx["type"] == "transaction" - - llm_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" - ] - assert len(llm_spans) > 0 + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + llm_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.text_completion" + ] + assert len(llm_spans) > 0 - llm_span = llm_spans[0] - assert llm_span["description"] == "text_completion gpt-3.5-turbo" - assert llm_span["data"]["gen_ai.system"] == "openai" - assert llm_span["data"]["gen_ai.function_id"] == "my-snazzy-pipeline" - assert llm_span["data"]["gen_ai.request.model"] == "gpt-3.5-turbo" - assert ( - llm_span["data"]["gen_ai.response.text"] - == "The capital of France is Paris." - ) - assert llm_span["data"]["gen_ai.usage.total_tokens"] == 25 - assert llm_span["data"]["gen_ai.usage.input_tokens"] == 10 - assert llm_span["data"]["gen_ai.usage.output_tokens"] == 15 + llm_span = llm_spans[0] + assert llm_span["name"] == "text_completion gpt-3.5-turbo" + assert llm_span["attributes"]["gen_ai.system"] == "openai" + assert llm_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" + assert llm_span["attributes"]["gen_ai.request.model"] == "gpt-3.5-turbo" + assert ( + llm_span["attributes"]["gen_ai.response.text"] + == "The capital of France is Paris." + ) + assert llm_span["attributes"]["gen_ai.usage.total_tokens"] == 25 + assert llm_span["attributes"]["gen_ai.usage.input_tokens"] == 10 + assert llm_span["attributes"]["gen_ai.usage.output_tokens"] == 15 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_chat_with_run_name( sentry_init, - capture_events, capture_items, get_model_response, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[ @@ -418,8 +375,7 @@ def test_langchain_chat_with_run_name( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) request_headers = {} @@ -448,64 +404,36 @@ def test_langchain_chat_with_run_name( temperature=0, openai_api_key="badkey", ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with patch.object( - llm.client._client._client, - "send", - return_value=model_response, - ) as _, start_transaction(): - llm.invoke( - "How many letters in the word eudca", - config={"run_name": "my-snazzy-pipeline"}, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - chat_spans = list( - x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat" - ) - assert len(chat_spans) == 1 - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_FUNCTION_ID] - == "my-snazzy-pipeline" + with patch.object( + llm.client._client._client, + "send", + return_value=model_response, + ): + llm.invoke( + "How many letters in the word eudca", + config={"run_name": "my-snazzy-pipeline"}, ) - else: - events = capture_events() - - with patch.object( - llm.client._client._client, - "send", - return_value=model_response, - ) as _, start_transaction(): - llm.invoke( - "How many letters in the word eudca", - config={"run_name": "my-snazzy-pipeline"}, - ) - tx = events[0] - - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - assert len(chat_spans) == 1 - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_FUNCTION_ID] == "my-snazzy-pipeline" - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + chat_spans = list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat") + assert len(chat_spans) == 1 + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_FUNCTION_ID] == "my-snazzy-pipeline" + ) @pytest.mark.skipif( ChatGoogleGenerativeAI is None, reason="Requires langchain-google-genai.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_multi_choice_response( sentry_init, - capture_events, capture_items, get_model_response, nonstreaming_multi_candidate_google_genai_model_response, - span_streaming, ): sentry_init( integrations=[ @@ -516,8 +444,7 @@ def test_langchain_multi_choice_response( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) model_response = get_model_response( @@ -530,64 +457,34 @@ def test_langchain_multi_choice_response( temperature=0, google_api_key="badkey", ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with patch.object( - llm.client._api_client._httpx_client, - "send", - return_value=model_response, - ) as _, start_transaction(): - llm.invoke( - "How many letters in the word eudca", - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - chat_spans = list( - x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat" - ) - assert len(chat_spans) == 1 - - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 25 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 - - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] - == 4 + with patch.object( + llm.client._api_client._httpx_client, + "send", + return_value=model_response, + ): + llm.invoke( + "How many letters in the word eudca", ) - else: - events = capture_events() - - with patch.object( - llm.client._api_client._httpx_client, - "send", - return_value=model_response, - ) as _, start_transaction(): - llm.invoke( - "How many letters in the word eudca", - ) - - tx = events[0] - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - assert len(chat_spans) == 1 + sentry_sdk.flush() + spans = [item.payload for item in items] + chat_spans = list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat") + assert len(chat_spans) == 1 - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 10 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 25 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30 + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 25 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 - assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4 + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4 + ) -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_tool_call_with_run_name( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[ @@ -598,46 +495,26 @@ def test_langchain_tool_call_with_run_name( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) - if span_streaming: - items = capture_items("span") + items = capture_items("span") - with start_transaction(): - get_word_length.invoke( - {"word": "eudca"}, - config={"run_name": "my-snazzy-pipeline"}, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - tool_spans = list( - x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.execute_tool" - ) - assert len(tool_spans) == 1 - assert ( - tool_spans[0]["attributes"][SPANDATA.GEN_AI_FUNCTION_ID] - == "my-snazzy-pipeline" - ) - else: - events = capture_events() - - with start_transaction(): - get_word_length.invoke( - {"word": "eudca"}, - config={"run_name": "my-snazzy-pipeline"}, - ) + get_word_length.invoke( + {"word": "eudca"}, + config={"run_name": "my-snazzy-pipeline"}, + ) - tx = events[0] - tool_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool") - assert len(tool_spans) == 1 - assert ( - tool_spans[0]["data"][SPANDATA.GEN_AI_FUNCTION_ID] == "my-snazzy-pipeline" - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + tool_spans = list( + x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.execute_tool" + ) + assert len(tool_spans) == 1 + assert ( + tool_spans[0]["attributes"][SPANDATA.GEN_AI_FUNCTION_ID] == "my-snazzy-pipeline" + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( LANGCHAIN_VERSION < (1,), reason="LangChain 1.0+ required (ONE AGENT refactor)", @@ -683,7 +560,6 @@ def test_langchain_tool_call_with_run_name( ) def test_langchain_create_agent( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, @@ -691,7 +567,6 @@ def test_langchain_create_agent( expected_system_instructions, get_model_response, nonstreaming_responses_model_response, - span_streaming, ): sentry_init( integrations=[ @@ -702,8 +577,7 @@ def test_langchain_create_agent( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) model_response = get_model_response( @@ -726,156 +600,87 @@ def test_langchain_create_agent( system_prompt=SystemMessage(content=system_instructions_content), name="word_length_agent", ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - llm.client._client._client, - "send", - return_value=model_response, - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - agent.invoke( - { - "messages": [ - HumanMessage( - content="Message demonstrating the absence of truncation." - ), - HumanMessage(content="How many letters in the word eudca"), - ], - }, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[2]["attributes"]["sentry.origin"] == "manual" - chat_spans = list( - x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) - assert len(chat_spans) == 1 - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - - assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" - assert chat_spans[0]["attributes"]["gen_ai.agent.name"] == "word_length_agent" - - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 - - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] - == 4 - ) - assert ( - chat_spans[0]["attributes"][ - SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS - ] - == 6 - ) - - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] - == 5 + with patch.object( + llm.client._client._client, + "send", + return_value=model_response, + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + agent.invoke( + { + "messages": [ + HumanMessage( + content="Message demonstrating the absence of truncation." + ), + HumanMessage(content="How many letters in the word eudca"), + ], + }, ) - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" - ) - - if send_default_pii and include_prompts: - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == "Hello, how can I help you?" - ) - - assert json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, - { - "role": "user", - "content": "How many letters in the word eudca", - }, - ] - - assert expected_system_instructions == json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get( - "attributes", {} - ) + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[2]["attributes"]["sentry.origin"] == "manual" + chat_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" + ) + assert len(chat_spans) == 1 + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - else: - events = capture_events() - - with patch.object( - llm.client._client._client, - "send", - return_value=model_response, - ) as _, start_transaction(): - agent.invoke( - { - "messages": [ - HumanMessage(content="How many letters in the word eudca"), - ], - }, - ) + assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" + assert chat_spans[0]["attributes"]["gen_ai.agent.name"] == "word_length_agent" - tx = events[0] - assert tx["type"] == "transaction" - assert tx["contexts"]["trace"]["origin"] == "manual" + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - assert len(chat_spans) == 1 - assert chat_spans[0]["origin"] == "auto.ai.langchain" + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4 + ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 6 + ) - assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat" - assert chat_spans[0]["data"]["gen_ai.agent.name"] == "word_length_agent" + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 5 + ) - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 10 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30 + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" - assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4 + if send_default_pii and include_prompts: assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] - == 6 + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == "Hello, how can I help you?" ) - assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 5 - - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" - - if send_default_pii and include_prompts: - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == "Hello, how can I help you?" - ) + assert json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "How many letters in the word eudca", + }, + ] - assert expected_system_instructions == json.loads( - chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( - "data", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) + assert expected_system_instructions == json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( LANGCHAIN_VERSION < (1,), reason="LangChain 1.0+ required (ONE AGENT refactor)", @@ -891,13 +696,11 @@ def test_langchain_create_agent( ) def test_tool_execution_span( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, nonstreaming_responses_tool_call_model_responses, - span_streaming, ): sentry_init( integrations=[ @@ -908,8 +711,7 @@ def test_tool_execution_span( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) responses = nonstreaming_responses_tool_call_model_responses( @@ -973,265 +775,129 @@ def test_tool_execution_span( tools=[get_word_length], name="word_length_agent", ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - agent.invoke( - { - "messages": [ - HumanMessage(content="How many letters in the word eudca"), - ], - }, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[4]["attributes"]["sentry.origin"] == "manual" - chat_spans = list( - x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) - assert len(chat_spans) == 2 - - tool_exec_spans = list( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" + with patch.object( + llm.client._client._client, + "send", + side_effect=[tool_response, final_response], + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + agent.invoke( + { + "messages": [ + HumanMessage(content="How many letters in the word eudca"), + ], + }, ) - assert len(tool_exec_spans) == 1 - tool_exec_span = tool_exec_spans[0] - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[4]["attributes"]["sentry.origin"] == "manual" + chat_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" + ) + assert len(chat_spans) == 2 - assert chat_spans[0]["attributes"]["gen_ai.agent.name"] == "word_length_agent" - assert chat_spans[1]["attributes"]["gen_ai.agent.name"] == "word_length_agent" - assert tool_exec_span["attributes"]["gen_ai.agent.name"] == "word_length_agent" + tool_exec_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" + ) + assert len(tool_exec_spans) == 1 + tool_exec_span = tool_exec_spans[0] + + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + + assert chat_spans[0]["attributes"]["gen_ai.agent.name"] == "word_length_agent" + assert chat_spans[1]["attributes"]["gen_ai.agent.name"] == "word_length_agent" + assert tool_exec_span["attributes"]["gen_ai.agent.name"] == "word_length_agent" + + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 69 + ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 31 + ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 10 + ) + assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] - == 69 - ) - assert ( - chat_spans[0]["attributes"][ - SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS - ] - == 31 - ) - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] - == 10 - ) - assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 69 + ) + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 10 + ) + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 11 + ) + assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat" - assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - assert ( - chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] - == 69 - ) + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert ( - chat_spans[1]["attributes"][ - SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS - ] - == 10 + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4-0613" ) assert ( - chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] - == 11 - ) - assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat" - - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-4-0613" - ) - assert ( - chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-4-0613" - ) - - if send_default_pii and include_prompts: - assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] - - assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( - "attributes", {} - ), ( - "Tool calls should be recorded when send_default_pii=True and include_prompts=True" - ) - tool_calls_data = chat_spans[0]["attributes"][ - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS - ] - assert isinstance(tool_calls_data, str) - assert "get_word_length" in tool_calls_data - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get( - "attributes", {} - ) - - # Verify tool calls are NOT recorded when PII is disabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "attributes", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "attributes", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["attributes"][ - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS - ] - assert "get_word_length" in tools_data - else: - events = capture_events() - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, start_transaction(): - agent.invoke( - { - "messages": [ - HumanMessage(content="How many letters in the word eudca"), - ], - }, - ) - - tx = events[0] - assert tx["type"] == "transaction" - assert tx["contexts"]["trace"]["origin"] == "manual" - - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - assert len(chat_spans) == 2 - tool_exec_spans = list( - x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool" + chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4-0613" ) - assert len(tool_exec_spans) == 1 - tool_exec_span = tool_exec_spans[0] - - assert chat_spans[0]["origin"] == "auto.ai.langchain" - assert chat_spans[1]["origin"] == "auto.ai.langchain" - assert tool_exec_span["origin"] == "auto.ai.langchain" + if send_default_pii and include_prompts: + assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] - assert chat_spans[0]["data"]["gen_ai.agent.name"] == "word_length_agent" - assert chat_spans[1]["data"]["gen_ai.agent.name"] == "word_length_agent" - assert tool_exec_span["data"]["gen_ai.agent.name"] == "word_length_agent" + assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 69 + # Verify tool calls are recorded when PII is enabled + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( + "attributes", {} + ), ( + "Tool calls should be recorded when send_default_pii=True and include_prompts=True" ) - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] - == 31 + tool_calls_data = chat_spans[0]["attributes"][ + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS + ] + assert isinstance(tool_calls_data, str) + assert "get_word_length" in tool_calls_data + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( + "attributes", {} ) - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 10 + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( + "attributes", {} ) - assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat" + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) - assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 - assert ( - chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 69 - ) - assert ( - chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] - == 10 + # Verify tool calls are NOT recorded when PII is disabled + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( + "attributes", {} + ), ( + f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " + f"and include_prompts={include_prompts}" ) - assert ( - chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 11 + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( + "attributes", {} + ), ( + f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " + f"and include_prompts={include_prompts}" ) - assert chat_spans[1]["data"]["gen_ai.system"] == "openai-chat" - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4-0613" - assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4-0613" + # Verify that available tools are always recorded regardless of PII settings + for chat_span in chat_spans: + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + assert "get_word_length" in tools_data - if send_default_pii and include_prompts: - assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT] - assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( - "data", {} - ), ( - "Tool calls should be recorded when send_default_pii=True and include_prompts=True" - ) - tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] - assert isinstance(tool_calls_data, str) - assert "get_word_length" in tool_calls_data - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {}) - - # Verify tool calls are NOT recorded when PII is disabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "data", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "data", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - assert "get_word_length" in tools_data - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -1242,14 +908,12 @@ def test_tool_execution_span( ) def test_langchain_openai_tools_agent_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, streaming_chat_completions_model_responses, - span_streaming, ): sentry_init( integrations=[ @@ -1260,8 +924,7 @@ def test_langchain_openai_tools_agent_no_prompts( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) prompt = ChatPromptTemplate.from_messages( @@ -1299,233 +962,114 @@ def test_langchain_openai_tools_agent_no_prompts( agent = create_openai_tools_agent(llm, [get_word_length], prompt) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - list( - agent_executor.invoke( - {"input": "How many letters in the word eudca"}, - {"run_name": "my-snazzy-pipeline"}, - ) + with patch.object( + llm.client._client._client, + "send", + side_effect=[tool_response, final_response], + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + list( + agent_executor.invoke( + {"input": "How many letters in the word eudca"}, + {"run_name": "my-snazzy-pipeline"}, ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[4]["attributes"]["sentry.origin"] == "manual" - invoke_agent_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" - ) - chat_spans = list( - x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) - tool_exec_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" ) - assert len(chat_spans) == 2 - - assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - - assert ( - invoke_agent_span["attributes"]["gen_ai.function_id"] - == "my-snazzy-pipeline" - ) + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[4]["attributes"]["sentry.origin"] == "manual" + invoke_agent_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" + ) + chat_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" + ) + tool_exec_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" + ) - # We can't guarantee anything about the "shape" of the langchain execution graph - assert ( - len( - list( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) - ) - > 0 - ) + assert len(chat_spans) == 2 - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - - if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: - assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" - ) - assert ( - chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" - ) + assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) + assert invoke_agent_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" - # Verify tool calls are NOT recorded when PII is disabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "attributes", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "attributes", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] + # We can't guarantee anything about the "shape" of the langchain execution graph + assert ( + len(list(x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat")) + > 0 + ) - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["attributes"][ - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS - ] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data - else: - events = capture_events() - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, start_transaction(): - list( - agent_executor.invoke( - {"input": "How many letters in the word eudca"}, - {"run_name": "my-snazzy-pipeline"}, - ) - ) + # Token usage is only available in newer versions of langchain (v0.2+) + # where usage_metadata is supported on AIMessageChunk + if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - tx = events[0] - assert tx["type"] == "transaction" - assert tx["contexts"]["trace"]["origin"] == "manual" + if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - invoke_agent_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent" + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" ) - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - tool_exec_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool" + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" ) - assert len(chat_spans) == 2 - - assert invoke_agent_span["origin"] == "auto.ai.langchain" - assert chat_spans[0]["origin"] == "auto.ai.langchain" - assert chat_spans[1]["origin"] == "auto.ai.langchain" - assert tool_exec_span["origin"] == "auto.ai.langchain" - - assert invoke_agent_span["data"]["gen_ai.function_id"] == "my-snazzy-pipeline" - - # We can't guarantee anything about the "shape" of the langchain execution graph - assert len(list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")) > 0 - - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]: - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 - - if "gen_ai.usage.input_tokens" in chat_spans[1]["data"]: - assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 - - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" - ) - assert ( - chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" - ) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) + + # Verify tool calls are NOT recorded when PII is disabled + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( + "attributes", {} + ), ( + f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " + f"and include_prompts={include_prompts}" + ) + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( + "attributes", {} + ), ( + f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " + f"and include_prompts={include_prompts}" + ) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {}) + # Verify finish_reasons is always an array of strings + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "function_call" + ] + assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "stop" + ] - # Verify tool calls are NOT recorded when PII is disabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "data", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "data", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" + # Verify that available tools are always recorded regardless of PII settings + for chat_span in chat_spans: + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + assert tools_data is not None, ( + "Available tools should always be recorded regardless of PII settings" ) - - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] - - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data + assert "get_word_length" in tools_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "system_instructions_content,expected_system_instructions", [ @@ -1571,14 +1115,12 @@ def test_langchain_openai_tools_agent_no_prompts( ) def test_langchain_openai_tools_agent( sentry_init, - capture_events, capture_items, system_instructions_content, expected_system_instructions, get_model_response, server_side_event_chunks, streaming_chat_completions_model_responses, - span_streaming, ): sentry_init( integrations=[ @@ -1589,8 +1131,7 @@ def test_langchain_openai_tools_agent( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) prompt = ChatPromptTemplate.from_messages( @@ -1628,240 +1169,128 @@ def test_langchain_openai_tools_agent( agent = create_openai_tools_agent(llm, [get_word_length], prompt) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - list( - agent_executor.stream( - { - "input": [ - "Message demonstrating the absence of truncation.", - "How many letters in the word eudca", - ] - } - ) - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[4]["attributes"]["sentry.origin"] == "manual" - invoke_agent_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" - ) - chat_spans = list( - x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) - tool_exec_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" - ) - - assert len(chat_spans) == 2 - - assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - - # We can't guarantee anything about the "shape" of the langchain execution graph - assert ( - len( - list( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) + with patch.object( + llm.client._client._client, + "send", + side_effect=[tool_response, final_response], + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + list( + agent_executor.stream( + { + "input": [ + "Message demonstrating the absence of truncation.", + "How many letters in the word eudca", + ] + } ) - > 0 ) - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - - if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: - assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" - ) - assert ( - chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" - ) - - assert "5" in chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] - assert 5 == int(tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_OUTPUT]) - - assert json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) == [ - { - "role": "user", - "content": "['Message demonstrating the absence of truncation.', 'How many letters in the word eudca']", - } - ] - - assert expected_system_instructions == json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - - assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[4]["attributes"]["sentry.origin"] == "manual" + invoke_agent_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" + ) + chat_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" + ) + tool_exec_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" + ) - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( - "attributes", {} - ), ( - "Tool calls should be recorded when send_default_pii=True and include_prompts=True" - ) - tool_calls_data = chat_spans[0]["attributes"][ - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS - ] + assert len(chat_spans) == 2 - assert isinstance(tool_calls_data, (list, str)) # Could be serialized - if isinstance(tool_calls_data, str): - assert "get_word_length" in tool_calls_data - elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0: - # Check if tool calls contain expected function name - tool_call_str = str(tool_calls_data) - assert "get_word_length" in tool_call_str - - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] + assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["attributes"][ - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS - ] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data - else: - events = capture_events() + # We can't guarantee anything about the "shape" of the langchain execution graph + assert ( + len(list(x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat")) + > 0 + ) - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, start_transaction(): - list(agent_executor.stream({"input": "How many letters in the word eudca"})) + # Token usage is only available in newer versions of langchain (v0.2+) + # where usage_metadata is supported on AIMessageChunk + if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - tx = events[0] - assert tx["type"] == "transaction" - assert tx["contexts"]["trace"]["origin"] == "manual" + if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - invoke_agent_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent" + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" ) - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - tool_exec_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool" + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" ) - assert len(chat_spans) == 2 + assert "5" in chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] + assert 5 == int(tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_OUTPUT]) - assert invoke_agent_span["origin"] == "auto.ai.langchain" - assert chat_spans[0]["origin"] == "auto.ai.langchain" - assert chat_spans[1]["origin"] == "auto.ai.langchain" - assert tool_exec_span["origin"] == "auto.ai.langchain" - - # We can't guarantee anything about the "shape" of the langchain execution graph - assert len(list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")) > 0 - - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]: - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 - - if "gen_ai.usage.input_tokens" in chat_spans[1]["data"]: - assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 - - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" - ) - assert ( - chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" - ) + assert json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": "['Message demonstrating the absence of truncation.', 'How many letters in the word eudca']", + } + ] - assert "5" in chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT] - assert 5 == int(tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_OUTPUT]) + assert expected_system_instructions == json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) - assert expected_system_instructions == json.loads( - chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) + assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + # Verify tool calls are recorded when PII is enabled + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("attributes", {}), ( + "Tool calls should be recorded when send_default_pii=True and include_prompts=True" + ) + tool_calls_data = chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + + assert isinstance(tool_calls_data, (list, str)) # Could be serialized + if isinstance(tool_calls_data, str): + assert "get_word_length" in tool_calls_data + elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0: + # Check if tool calls contain expected function name + tool_call_str = str(tool_calls_data) + assert "get_word_length" in tool_call_str + + # Verify finish_reasons is always an array of strings + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "function_call" + ] + assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "stop" + ] - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("data", {}), ( - "Tool calls should be recorded when send_default_pii=True and include_prompts=True" + # Verify that available tools are always recorded regardless of PII settings + for chat_span in chat_spans: + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + assert tools_data is not None, ( + "Available tools should always be recorded regardless of PII settings" ) - tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] - - assert isinstance(tool_calls_data, (list, str)) # Could be serialized - if isinstance(tool_calls_data, str): - assert "get_word_length" in tool_calls_data - elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0: - # Check if tool calls contain expected function name - tool_call_str = str(tool_calls_data) - assert "get_word_length" in tool_call_str - - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] - - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data + assert "get_word_length" in tools_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_openai_tools_agent_with_config( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, streaming_chat_completions_model_responses, - span_streaming, ): sentry_init( integrations=[ @@ -1872,8 +1301,7 @@ def test_langchain_openai_tools_agent_with_config( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) prompt = ChatPromptTemplate.from_messages( @@ -1913,58 +1341,28 @@ def test_langchain_openai_tools_agent_with_config( ) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - list( - agent_executor.invoke( - {"input": "How many letters in the word eudca"}, - ) + with patch.object( + llm.client._client._client, + "send", + side_effect=[tool_response, final_response], + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + list( + agent_executor.invoke( + {"input": "How many letters in the word eudca"}, ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[4]["attributes"]["sentry.origin"] == "manual" - invoke_agent_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" - ) - assert ( - invoke_agent_span["attributes"]["gen_ai.function_id"] - == "my-snazzy-pipeline" ) - else: - events = capture_events() - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, start_transaction(): - list( - agent_executor.invoke( - {"input": "How many letters in the word eudca"}, - ) - ) - tx = events[0] - assert tx["type"] == "transaction" - assert tx["contexts"]["trace"]["origin"] == "manual" - - invoke_agent_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent" - ) - assert invoke_agent_span["data"]["gen_ai.function_id"] == "my-snazzy-pipeline" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[4]["attributes"]["sentry.origin"] == "manual" + invoke_agent_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" + ) + assert invoke_agent_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -1975,14 +1373,12 @@ def test_langchain_openai_tools_agent_with_config( ) def test_langchain_openai_tools_agent_stream_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, streaming_chat_completions_model_responses, - span_streaming, ): sentry_init( integrations=[ @@ -1993,8 +1389,7 @@ def test_langchain_openai_tools_agent_stream_no_prompts( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) prompt = ChatPromptTemplate.from_messages( @@ -2032,236 +1427,117 @@ def test_langchain_openai_tools_agent_stream_no_prompts( agent = create_openai_tools_agent(llm, [get_word_length], prompt) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - list( - agent_executor.stream( - {"input": "How many letters in the word eudca"}, - {"run_name": "my-snazzy-pipeline"}, - ) - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[4]["attributes"]["sentry.origin"] == "manual" - invoke_agent_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" - ) - chat_spans = list( - x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) - tool_exec_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" - ) - - assert len(chat_spans) == 2 - - assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - - assert ( - invoke_agent_span["attributes"]["gen_ai.function_id"] - == "my-snazzy-pipeline" - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - # We can't guarantee anything about the "shape" of the langchain execution graph - assert ( - len( - list( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) + with patch.object( + llm.client._client._client, + "send", + side_effect=[tool_response, final_response], + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + list( + agent_executor.stream( + {"input": "How many letters in the word eudca"}, + {"run_name": "my-snazzy-pipeline"}, ) - > 0 ) - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - - if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: - assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" - ) - assert ( - chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" - ) + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[4]["attributes"]["sentry.origin"] == "manual" + invoke_agent_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" + ) + chat_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" + ) + tool_exec_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" + ) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) + assert len(chat_spans) == 2 - # Verify tool calls are NOT recorded when PII is disabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "attributes", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "attributes", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) + assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] + assert invoke_agent_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["attributes"][ - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS - ] + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + # We can't guarantee anything about the "shape" of the langchain execution graph + assert ( + len(list(x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat")) + > 0 + ) - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data - else: - events = capture_events() - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, start_transaction(): - list( - agent_executor.stream( - {"input": "How many letters in the word eudca"}, - {"run_name": "my-snazzy-pipeline"}, - ) - ) + # Token usage is only available in newer versions of langchain (v0.2+) + # where usage_metadata is supported on AIMessageChunk + if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - tx = events[0] - assert tx["type"] == "transaction" - assert tx["contexts"]["trace"]["origin"] == "manual" + if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - invoke_agent_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent" + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" ) - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - tool_exec_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool" + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" ) - assert len(chat_spans) == 2 - - assert invoke_agent_span["origin"] == "auto.ai.langchain" - assert chat_spans[0]["origin"] == "auto.ai.langchain" - assert chat_spans[1]["origin"] == "auto.ai.langchain" - assert tool_exec_span["origin"] == "auto.ai.langchain" - - assert invoke_agent_span["data"]["gen_ai.function_id"] == "my-snazzy-pipeline" - - # We can't guarantee anything about the "shape" of the langchain execution graph - assert len(list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")) > 0 - - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]: - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 - - if "gen_ai.usage.input_tokens" in chat_spans[1]["data"]: - assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) + + # Verify tool calls are NOT recorded when PII is disabled + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( + "attributes", {} + ), ( + f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " + f"and include_prompts={include_prompts}" + ) + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( + "attributes", {} + ), ( + f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " + f"and include_prompts={include_prompts}" + ) - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" - ) - assert ( - chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" - ) + # Verify finish_reasons is always an array of strings + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "function_call" + ] + assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "stop" + ] - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {}) + # Verify that available tools are always recorded regardless of PII settings + for chat_span in chat_spans: + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - # Verify tool calls are NOT recorded when PII is disabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "data", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" + assert tools_data is not None, ( + "Available tools should always be recorded regardless of PII settings" ) - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "data", {} - ), ( - f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " - f"and include_prompts={include_prompts}" - ) - - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] + assert "get_word_length" in tools_data - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "system_instructions_content,expected_system_instructions", [ @@ -2308,14 +1584,12 @@ def test_langchain_openai_tools_agent_stream_no_prompts( ) def test_langchain_openai_tools_agent_stream( sentry_init, - capture_events, capture_items, system_instructions_content, expected_system_instructions, get_model_response, server_side_event_chunks, streaming_chat_completions_model_responses, - span_streaming, ): sentry_init( integrations=[ @@ -2326,8 +1600,7 @@ def test_langchain_openai_tools_agent_stream( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) prompt = ChatPromptTemplate.from_messages( @@ -2354,263 +1627,142 @@ def test_langchain_openai_tools_agent_stream( server_side_event_chunks( next(model_responses), include_event_type=False, - ) - ) - - llm = ChatOpenAI( - model_name="gpt-3.5-turbo", - temperature=0, - openai_api_key="badkey", - ) - agent = create_openai_tools_agent(llm, [get_word_length], prompt) - - agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) - - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - list( - agent_executor.stream( - { - "input": [ - "Message demonstrating the absence of truncation.", - "How many letters in the word eudca", - ] - }, - {"run_name": "my-snazzy-pipeline"}, - ) - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[4]["attributes"]["sentry.origin"] == "manual" - invoke_agent_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" - ) - chat_spans = list( - x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) - tool_exec_span = next( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" - ) - - assert len(chat_spans) == 2 - - assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - - assert ( - invoke_agent_span["attributes"]["gen_ai.function_id"] - == "my-snazzy-pipeline" - ) - - # We can't guarantee anything about the "shape" of the langchain execution graph - assert ( - len( - list( - x - for x in spans - if x["attributes"].get("sentry.op") == "gen_ai.chat" - ) - ) - > 0 - ) - - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - - if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: - assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" - ) - assert ( - chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" - ) - - assert "5" in chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] - assert 5 == int(tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_OUTPUT]) - - assert json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) == [ - { - "role": "user", - "content": "['Message demonstrating the absence of truncation.', 'How many letters in the word eudca']", - } - ] - - assert expected_system_instructions == json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - - assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( - "attributes", {} - ), ( - "Tool calls should be recorded when send_default_pii=True and include_prompts=True" - ) - tool_calls_data = chat_spans[0]["attributes"][ - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS - ] + ) + ) - assert isinstance(tool_calls_data, (list, str)) # Could be serialized - if isinstance(tool_calls_data, str): - assert "get_word_length" in tool_calls_data - elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0: - # Check if tool calls contain expected function name - tool_call_str = str(tool_calls_data) - assert "get_word_length" in tool_call_str - - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] + llm = ChatOpenAI( + model_name="gpt-3.5-turbo", + temperature=0, + openai_api_key="badkey", + ) + agent = create_openai_tools_agent(llm, [get_word_length], prompt) - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["attributes"][ - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS - ] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data - else: - events = capture_events() - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, start_transaction(): - list( - agent_executor.stream( - {"input": "How many letters in the word eudca"}, - {"run_name": "my-snazzy-pipeline"}, - ) + agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + items = capture_items("transaction", "span") + + with patch.object( + llm.client._client._client, + "send", + side_effect=[tool_response, final_response], + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + list( + agent_executor.stream( + { + "input": [ + "Message demonstrating the absence of truncation.", + "How many letters in the word eudca", + ] + }, + {"run_name": "my-snazzy-pipeline"}, ) + ) - tx = events[0] - assert tx["type"] == "transaction" - assert tx["contexts"]["trace"]["origin"] == "manual" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[4]["attributes"]["sentry.origin"] == "manual" + invoke_agent_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" + ) + chat_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" + ) + tool_exec_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" + ) - invoke_agent_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent" - ) - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - tool_exec_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool" - ) + assert len(chat_spans) == 2 - assert len(chat_spans) == 2 + assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert invoke_agent_span["origin"] == "auto.ai.langchain" - assert chat_spans[0]["origin"] == "auto.ai.langchain" - assert chat_spans[1]["origin"] == "auto.ai.langchain" - assert tool_exec_span["origin"] == "auto.ai.langchain" + assert invoke_agent_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" + + # We can't guarantee anything about the "shape" of the langchain execution graph + assert ( + len(list(x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat")) + > 0 + ) - assert invoke_agent_span["data"]["gen_ai.function_id"] == "my-snazzy-pipeline" + # Token usage is only available in newer versions of langchain (v0.2+) + # where usage_metadata is supported on AIMessageChunk + if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - # We can't guarantee anything about the "shape" of the langchain execution graph - assert len(list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")) > 0 + if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]: - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" + ) + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" + ) - if "gen_ai.usage.input_tokens" in chat_spans[1]["data"]: - assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 + assert "5" in chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] + assert 5 == int(tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_OUTPUT]) - if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): - assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" - ) - assert ( - chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" - ) + assert json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": "['Message demonstrating the absence of truncation.', 'How many letters in the word eudca']", + } + ] - assert "5" in chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT] - assert 5 == int(tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_OUTPUT]) + assert expected_system_instructions == json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) - assert expected_system_instructions == json.loads( - chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) + assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + # Verify tool calls are recorded when PII is enabled + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("attributes", {}), ( + "Tool calls should be recorded when send_default_pii=True and include_prompts=True" + ) + tool_calls_data = chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + + assert isinstance(tool_calls_data, (list, str)) # Could be serialized + if isinstance(tool_calls_data, str): + assert "get_word_length" in tool_calls_data + elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0: + # Check if tool calls contain expected function name + tool_call_str = str(tool_calls_data) + assert "get_word_length" in tool_call_str + + # Verify finish_reasons is always an array of strings + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "function_call" + ] + assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "stop" + ] - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("data", {}), ( - "Tool calls should be recorded when send_default_pii=True and include_prompts=True" + # Verify that available tools are always recorded regardless of PII settings + for chat_span in chat_spans: + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + assert tools_data is not None, ( + "Available tools should always be recorded regardless of PII settings" ) - tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] - assert isinstance(tool_calls_data, (list, str)) # Could be serialized - if isinstance(tool_calls_data, str): - assert "get_word_length" in tool_calls_data - elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0: - # Check if tool calls contain expected function name - tool_call_str = str(tool_calls_data) - assert "get_word_length" in tool_call_str - - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] - - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data + assert "get_word_length" in tools_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_openai_tools_agent_stream_with_config( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, streaming_chat_completions_model_responses, - span_streaming, ): sentry_init( integrations=[ @@ -2621,8 +1773,7 @@ def test_langchain_openai_tools_agent_stream_with_config( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) prompt = ChatPromptTemplate.from_messages( @@ -2662,61 +1813,31 @@ def test_langchain_openai_tools_agent_stream_with_config( ) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, sentry_sdk.traces.start_span(name="custom parent"): - list( - agent_executor.stream( - {"input": "How many letters in the word eudca"}, - ) + with patch.object( + llm.client._client._client, + "send", + side_effect=[tool_response, final_response], + ) as _, sentry_sdk.traces.start_span(name="custom parent"): + list( + agent_executor.stream( + {"input": "How many letters in the word eudca"}, ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[4]["attributes"]["sentry.origin"] == "manual" - invoke_agent_span = next( - x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.invoke_agent" - ) - assert ( - invoke_agent_span["attributes"]["gen_ai.function_id"] - == "my-snazzy-pipeline" ) - else: - events = capture_events() - - with patch.object( - llm.client._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _, start_transaction(): - list( - agent_executor.stream( - {"input": "How many letters in the word eudca"}, - ) - ) - tx = events[0] - assert tx["type"] == "transaction" - assert tx["contexts"]["trace"]["origin"] == "manual" - - invoke_agent_span = next( - x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent" - ) - assert invoke_agent_span["data"]["gen_ai.function_id"] == "my-snazzy-pipeline" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[4]["attributes"]["sentry.origin"] == "manual" + invoke_agent_span = next( + x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.invoke_agent" + ) + assert invoke_agent_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_error( sentry_init, - capture_events, capture_items, - span_streaming, ): class MockOpenAI(ChatOpenAI): def _stream( @@ -2740,8 +1861,7 @@ def _llm_type(self) -> str: disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) prompt = ChatPromptTemplate.from_messages( @@ -2762,30 +1882,18 @@ def _llm_type(self) -> str: agent = create_openai_tools_agent(llm, [get_word_length], prompt) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + items = capture_items("event") - if span_streaming: - items = capture_items("event") - - with start_transaction(), pytest.raises(ValueError): - list(agent_executor.stream({"input": "How many letters in the word eudca"})) - - (error,) = (item.payload for item in items) - else: - events = capture_events() - - with start_transaction(), pytest.raises(ValueError): - list(agent_executor.stream({"input": "How many letters in the word eudca"})) + with pytest.raises(ValueError): + list(agent_executor.stream({"input": "How many letters in the word eudca"})) - error = events[0] + (error,) = (item.payload for item in items) assert error["level"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_status_error( sentry_init, - capture_events, capture_items, - span_streaming, ): class MockOpenAI(ChatOpenAI): def _stream( @@ -2808,83 +1916,37 @@ def _llm_type(self) -> str: integrations=[LangchainIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("event", "transaction", "span") - if span_streaming: - items = capture_items("event", "transaction", "span") - - with start_transaction(name="test"): - prompt = ChatPromptTemplate.from_messages( - [ - ( - "system", - "You are very powerful assistant, but don't know current events", - ), - ("user", "{input}"), - MessagesPlaceholder(variable_name="agent_scratchpad"), - ] - ) - llm = MockOpenAI( - model_name="gpt-3.5-turbo", - temperature=0, - openai_api_key="badkey", - ) - agent = create_openai_tools_agent(llm, [get_word_length], prompt) - - agent_executor = AgentExecutor( - agent=agent, tools=[get_word_length], verbose=True - ) - - with pytest.raises(ValueError): - list( - agent_executor.stream( - {"input": "How many letters in the word eudca"} - ) - ) - - (error,) = (item.payload for item in items if item.type == "event") - assert error["level"] == "error" - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["status"] == "error" - else: - events = capture_events() - - with start_transaction(name="test"): - prompt = ChatPromptTemplate.from_messages( - [ - ( - "system", - "You are very powerful assistant, but don't know current events", - ), - ("user", "{input}"), - MessagesPlaceholder(variable_name="agent_scratchpad"), - ] - ) - llm = MockOpenAI( - model_name="gpt-3.5-turbo", - temperature=0, - openai_api_key="badkey", - ) - agent = create_openai_tools_agent(llm, [get_word_length], prompt) + prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + "You are very powerful assistant, but don't know current events", + ), + ("user", "{input}"), + MessagesPlaceholder(variable_name="agent_scratchpad"), + ] + ) + llm = MockOpenAI( + model_name="gpt-3.5-turbo", + temperature=0, + openai_api_key="badkey", + ) + agent = create_openai_tools_agent(llm, [get_word_length], prompt) - agent_executor = AgentExecutor( - agent=agent, tools=[get_word_length], verbose=True - ) + agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) - with pytest.raises(ValueError): - list( - agent_executor.stream( - {"input": "How many letters in the word eudca"} - ) - ) + with pytest.raises(ValueError): + list(agent_executor.stream({"input": "How many letters in the word eudca"})) - (error, transaction) = events - assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" + (error,) = (item.payload for item in items if item.type == "event") + assert error["level"] == "error" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["status"] == "error" def test_manual_callback_no_duplication(sentry_init): @@ -2935,7 +1997,6 @@ def _identifying_params(self): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], _experiments={"gen_ai_as_v2_spans": True}, - stream_gen_ai_spans=False, ) # Create a manual SentryLangchainCallback @@ -2976,7 +2037,6 @@ def test_langchain_callback_manager(sentry_init): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - stream_gen_ai_spans=False, ) local_manager = BaseCallbackManager(handlers=[]) @@ -3010,7 +2070,6 @@ def test_langchain_callback_manager_with_sentry_callback(sentry_init): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - stream_gen_ai_spans=False, ) sentry_callback = SentryLangchainCallback(False) local_manager = BaseCallbackManager(handlers=[sentry_callback]) @@ -3044,7 +2103,6 @@ def test_langchain_callback_list(sentry_init): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - stream_gen_ai_spans=False, ) local_callbacks = [] @@ -3078,7 +2136,6 @@ def test_langchain_callback_list_existing_callback(sentry_init): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - stream_gen_ai_spans=False, ) sentry_callback = SentryLangchainCallback(False) local_callbacks = [sentry_callback] @@ -3107,12 +2164,9 @@ def test_langchain_callback_list_existing_callback(sentry_init): assert handler is sentry_callback -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_message_role_mapping( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that message roles are properly normalized in langchain integration.""" @@ -3147,8 +2201,7 @@ def _llm_type(self) -> str: disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) prompt = ChatPromptTemplate.from_messages( @@ -3171,112 +2224,54 @@ def _llm_type(self) -> str: test_input = "Hello, how are you?" message_data_found = False - if span_streaming: - items = capture_items("span") - - with start_transaction(): - list(agent_executor.stream({"input": test_input})) - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find spans with gen_ai operation that should have message data - gen_ai_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op", "").startswith("gen_ai") - ] - - # Check if any span has message data with normalized roles - for span in gen_ai_spans: - span_data = span.get("attributes", {}) - if SPANDATA.GEN_AI_REQUEST_MESSAGES in span_data: - message_data_found = True - messages_data = span_data[SPANDATA.GEN_AI_REQUEST_MESSAGES] - - # Parse the message data (might be JSON string) - if isinstance(messages_data, str): - try: - messages = json.loads(messages_data) - except json.JSONDecodeError: - # If not valid JSON, skip this assertion - continue - else: - messages = messages_data - - # Verify that the input message is present and contains the test input - assert isinstance(messages, list) - assert len(messages) > 0 - - # The test input should be in one of the messages - input_found = False - for msg in messages: - if isinstance(msg, dict) and test_input in str( - msg.get("content", "") - ): - input_found = True - break - elif isinstance(msg, str) and test_input in msg: - input_found = True - break - - assert input_found, ( - f"Test input '{test_input}' not found in messages: {messages}" - ) - break - else: - events = capture_events() - - with start_transaction(): - list(agent_executor.stream({"input": test_input})) + items = capture_items("span") - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" + list(agent_executor.stream({"input": test_input})) - # Find spans with gen_ai operation that should have message data - gen_ai_spans = [ - span - for span in tx.get("spans", []) - if span.get("op", "").startswith("gen_ai") - ] + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find spans with gen_ai operation that should have message data + gen_ai_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op", "").startswith("gen_ai") + ] - # Check if any span has message data with normalized roles - for span in gen_ai_spans: - span_data = span.get("data", {}) - if SPANDATA.GEN_AI_REQUEST_MESSAGES in span_data: - message_data_found = True - messages_data = span_data[SPANDATA.GEN_AI_REQUEST_MESSAGES] - - # Parse the message data (might be JSON string) - if isinstance(messages_data, str): - try: - messages = json.loads(messages_data) - except json.JSONDecodeError: - # If not valid JSON, skip this assertion - continue - else: - messages = messages_data - - # Verify that the input message is present and contains the test input - assert isinstance(messages, list) - assert len(messages) > 0 - - # The test input should be in one of the messages - input_found = False - for msg in messages: - if isinstance(msg, dict) and test_input in str( - msg.get("content", "") - ): - input_found = True - break - elif isinstance(msg, str) and test_input in msg: - input_found = True - break - - assert input_found, ( - f"Test input '{test_input}' not found in messages: {messages}" - ) - break + # Check if any span has message data with normalized roles + for span in gen_ai_spans: + span_data = span.get("attributes", {}) + if SPANDATA.GEN_AI_REQUEST_MESSAGES in span_data: + message_data_found = True + messages_data = span_data[SPANDATA.GEN_AI_REQUEST_MESSAGES] + + # Parse the message data (might be JSON string) + if isinstance(messages_data, str): + try: + messages = json.loads(messages_data) + except json.JSONDecodeError: + # If not valid JSON, skip this assertion + continue + else: + messages = messages_data + + # Verify that the input message is present and contains the test input + assert isinstance(messages, list) + assert len(messages) > 0 + + # The test input should be in one of the messages + input_found = False + for msg in messages: + if isinstance(msg, dict) and test_input in str(msg.get("content", "")): + input_found = True + break + elif isinstance(msg, str) and test_input in msg: + input_found = True + break + + assert input_found, ( + f"Test input '{test_input}' not found in messages: {messages}" + ) + break # The message role mapping functionality is primarily tested through the normalization # that happens in the integration code. The fact that we can capture and process @@ -3325,87 +2320,6 @@ def test_langchain_message_role_normalization_units(): assert normalized[5] == "string message" # String message unchanged -def test_langchain_message_truncation(sentry_init, capture_events): - """Test that large messages are truncated properly in Langchain integration.""" - from langchain_core.outputs import Generation, LLMResult - - sentry_init( - integrations=[LangchainIntegration(include_prompts=True)], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - events = capture_events() - - callback = SentryLangchainCallback(include_prompts=True) - - run_id = "12345678-1234-1234-1234-123456789012" - serialized = {"_type": "openai-chat", "model_name": "gpt-3.5-turbo"} - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - prompts = [ - "small message 1", - large_content, - large_content, - "small message 4", - "small message 5", - ] - - with start_transaction(): - callback.on_llm_start( - serialized=serialized, - prompts=prompts, - run_id=run_id, - name="my_pipeline", - invocation_params={ - "temperature": 0.7, - "max_tokens": 100, - "model": "gpt-3.5-turbo", - }, - ) - - response = LLMResult( - generations=[[Generation(text="The response")]], - llm_output={ - "token_usage": { - "total_tokens": 25, - "prompt_tokens": 10, - "completion_tokens": 15, - } - }, - ) - callback.on_llm_end(response=response, run_id=run_id) - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - llm_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" - ] - assert len(llm_spans) > 0 - - llm_span = llm_spans[0] - assert llm_span["data"]["gen_ai.operation.name"] == "text_completion" - assert llm_span["data"][SPANDATA.GEN_AI_FUNCTION_ID] == "my_pipeline" - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in llm_span["data"] - messages_data = llm_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -3417,11 +2331,9 @@ def test_langchain_message_truncation(sentry_init, capture_events): ) def test_langchain_embeddings_sync( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): """Test that sync embedding methods (embed_documents, embed_query) are properly traced.""" try: @@ -3434,129 +2346,66 @@ def test_langchain_embeddings_sync( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, - ) - if span_streaming: - items = capture_items("span") - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ) as mock_embed_documents: - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) + trace_lifecycle="stream", + ) + items = capture_items("span") - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() + # Mock the actual API call + with mock.patch.object( + OpenAIEmbeddings, + "embed_documents", + wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], + ) as mock_embed_documents: + embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" + ) - with start_transaction(name="test_embeddings"): - # Test embed_documents - result = embeddings.embed_documents(["Hello world", "Test document"]) + # Force setup to re-run to ensure our mock is wrapped + LangchainIntegration.setup_once() - assert len(result) == 2 - mock_embed_documents.assert_called_once() + # Test embed_documents + result = embeddings.embed_documents(["Hello world", "Test document"]) - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find embeddings span - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 + assert len(result) == 2 + mock_embed_documents.assert_called_once() - embeddings_span = embeddings_spans[0] - assert embeddings_span["name"] == "embeddings text-embedding-ada-002" - assert embeddings_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["attributes"]["gen_ai.request.model"] - == "text-embedding-ada-002" - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find embeddings span + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + assert len(embeddings_spans) == 1 + + embeddings_span = embeddings_spans[0] + assert embeddings_span["name"] == "embeddings text-embedding-ada-002" + assert embeddings_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert ( + embeddings_span["attributes"]["gen_ai.request.model"] + == "text-embedding-ada-002" + ) - # Check if input is captured based on PII settings - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] - input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + # Check if input is captured based on PII settings + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] + input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - # Could be serialized as string - if isinstance(input_data, str): - assert "Hello world" in input_data - assert "Test document" in input_data - else: - assert "Hello world" in input_data - assert "Test document" in input_data + # Could be serialized as string + if isinstance(input_data, str): + assert "Hello world" in input_data + assert "Test document" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "attributes", {} - ) + assert "Hello world" in input_data + assert "Test document" in input_data else: - events = capture_events() - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ) as mock_embed_documents: - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() - - with start_transaction(name="test_embeddings"): - # Test embed_documents - result = embeddings.embed_documents(["Hello world", "Test document"]) - - assert len(result) == 2 - mock_embed_documents.assert_called_once() - - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - - # Find embeddings span - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 - - embeddings_span = embeddings_spans[0] - assert embeddings_span["description"] == "embeddings text-embedding-ada-002" - assert embeddings_span["origin"] == "auto.ai.langchain" - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( + "attributes", {} ) - # Check if input is captured based on PII settings - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] - input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - # Could be serialized as string - if isinstance(input_data, str): - assert "Hello world" in input_data - assert "Test document" in input_data - else: - assert "Hello world" in input_data - assert "Test document" in input_data - else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "data", {} - ) - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -3566,11 +2415,9 @@ def test_langchain_embeddings_sync( ) def test_langchain_embeddings_embed_query( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): """Test that embed_query method is properly traced.""" try: @@ -3583,119 +2430,61 @@ def test_langchain_embeddings_embed_query( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, - ) - if span_streaming: - items = capture_items("span") - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "embed_query", - wraps=lambda self, text: [0.1, 0.2, 0.3], - ) as mock_embed_query: - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() - - with start_transaction(name="test_embeddings_query"): - result = embeddings.embed_query("What is the capital of France?") - - assert len(result) == 3 - mock_embed_query.assert_called_once() - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find embeddings span - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 + trace_lifecycle="stream", + ) + items = capture_items("span") - embeddings_span = embeddings_spans[0] - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["attributes"]["gen_ai.request.model"] - == "text-embedding-ada-002" + # Mock the actual API call + with mock.patch.object( + OpenAIEmbeddings, + "embed_query", + wraps=lambda self, text: [0.1, 0.2, 0.3], + ) as mock_embed_query: + embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" ) - # Check if input is captured based on PII settings - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] - input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - - # Could be serialized as string - if isinstance(input_data, str): - assert "What is the capital of France?" in input_data - else: - assert "What is the capital of France?" in input_data - else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "attributes", {} - ) - else: - events = capture_events() - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "embed_query", - wraps=lambda self, text: [0.1, 0.2, 0.3], - ) as mock_embed_query: - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() + # Force setup to re-run to ensure our mock is wrapped + LangchainIntegration.setup_once() - with start_transaction(name="test_embeddings_query"): - result = embeddings.embed_query("What is the capital of France?") + result = embeddings.embed_query("What is the capital of France?") - assert len(result) == 3 - mock_embed_query.assert_called_once() + assert len(result) == 3 + mock_embed_query.assert_called_once() - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find embeddings span + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + assert len(embeddings_spans) == 1 - # Find embeddings span - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 + embeddings_span = embeddings_spans[0] + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert ( + embeddings_span["attributes"]["gen_ai.request.model"] + == "text-embedding-ada-002" + ) - embeddings_span = embeddings_spans[0] - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" - ) + # Check if input is captured based on PII settings + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] + input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - # Check if input is captured based on PII settings - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] - input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - # Could be serialized as string - if isinstance(input_data, str): - assert "What is the capital of France?" in input_data - else: - assert "What is the capital of France?" in input_data + # Could be serialized as string + if isinstance(input_data, str): + assert "What is the capital of France?" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "data", {} - ) + assert "What is the capital of France?" in input_data + else: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( + "attributes", {} + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -3706,11 +2495,9 @@ def test_langchain_embeddings_embed_query( @pytest.mark.asyncio async def test_langchain_embeddings_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): """Test that async embedding methods (aembed_documents, aembed_query) are properly traced.""" try: @@ -3723,146 +2510,73 @@ async def test_langchain_embeddings_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) async def mock_aembed_documents(self, texts): return [[0.1, 0.2, 0.3] for _ in texts] - if span_streaming: - items = capture_items("span") - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "aembed_documents", - wraps=mock_aembed_documents, - ) as mock_aembed: - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() - - with start_transaction(name="test_async_embeddings"): - result = await embeddings.aembed_documents( - ["Async hello", "Async test document"] - ) - - assert len(result) == 2 - mock_aembed.assert_called_once() - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find embeddings span - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 - - embeddings_span = embeddings_spans[0] - assert embeddings_span["name"] == "embeddings text-embedding-ada-002" - assert embeddings_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["attributes"]["gen_ai.request.model"] - == "text-embedding-ada-002" - ) - - # Check if input is captured based on PII settings - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] - input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - - # Could be serialized as string - if isinstance(input_data, str): - assert ( - "Async hello" in input_data or "Async test document" in input_data - ) - else: - assert ( - "Async hello" in input_data or "Async test document" in input_data - ) - else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "attributes", {} - ) - - else: - events = capture_events() - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "aembed_documents", - wraps=mock_aembed_documents, - ) as mock_aembed: - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) + items = capture_items("span") - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() + # Mock the actual API call + with mock.patch.object( + OpenAIEmbeddings, + "aembed_documents", + wraps=mock_aembed_documents, + ) as mock_aembed: + embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" + ) - with start_transaction(name="test_async_embeddings"): - result = await embeddings.aembed_documents( - ["Async hello", "Async test document"] - ) + # Force setup to re-run to ensure our mock is wrapped + LangchainIntegration.setup_once() - assert len(result) == 2 - mock_aembed.assert_called_once() + result = await embeddings.aembed_documents( + ["Async hello", "Async test document"] + ) - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" + assert len(result) == 2 + mock_aembed.assert_called_once() - # Find embeddings span - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find embeddings span + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + assert len(embeddings_spans) == 1 + + embeddings_span = embeddings_spans[0] + assert embeddings_span["name"] == "embeddings text-embedding-ada-002" + assert embeddings_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert ( + embeddings_span["attributes"]["gen_ai.request.model"] + == "text-embedding-ada-002" + ) - embeddings_span = embeddings_spans[0] - assert embeddings_span["description"] == "embeddings text-embedding-ada-002" - assert embeddings_span["origin"] == "auto.ai.langchain" - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" - ) + # Check if input is captured based on PII settings + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] + input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - # Check if input is captured based on PII settings - if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] - input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - # Could be serialized as string - if isinstance(input_data, str): - assert ( - "Async hello" in input_data or "Async test document" in input_data - ) - else: - assert ( - "Async hello" in input_data or "Async test document" in input_data - ) + # Could be serialized as string + if isinstance(input_data, str): + assert "Async hello" in input_data or "Async test document" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "data", {} - ) + assert "Async hello" in input_data or "Async test document" in input_data + else: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( + "attributes", {} + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_langchain_embeddings_aembed_query( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that aembed_query method is properly traced.""" try: @@ -3875,99 +2589,52 @@ async def test_langchain_embeddings_aembed_query( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) async def mock_aembed_query(self, text): return [0.1, 0.2, 0.3] - if span_streaming: - items = capture_items("span") - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "aembed_query", - wraps=mock_aembed_query, - ) as mock_aembed: - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() + items = capture_items("span") - with start_transaction(name="test_async_embeddings_query"): - result = await embeddings.aembed_query("Async query test") - - assert len(result) == 3 - mock_aembed.assert_called_once() - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find embeddings span - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 - - embeddings_span = embeddings_spans[0] - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["attributes"]["gen_ai.request.model"] - == "text-embedding-ada-002" + # Mock the actual API call + with mock.patch.object( + OpenAIEmbeddings, + "aembed_query", + wraps=mock_aembed_query, + ) as mock_aembed: + embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" ) - # Check if input is captured - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] - input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - else: - events = capture_events() - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "aembed_query", - wraps=mock_aembed_query, - ) as mock_aembed: - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() - - with start_transaction(name="test_async_embeddings_query"): - result = await embeddings.aembed_query("Async query test") + # Force setup to re-run to ensure our mock is wrapped + LangchainIntegration.setup_once() - assert len(result) == 3 - mock_aembed.assert_called_once() + result = await embeddings.aembed_query("Async query test") - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" + assert len(result) == 3 + mock_aembed.assert_called_once() - # Find embeddings span - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find embeddings span + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + assert len(embeddings_spans) == 1 - embeddings_span = embeddings_spans[0] - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" - ) + embeddings_span = embeddings_spans[0] + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert ( + embeddings_span["attributes"]["gen_ai.request.model"] + == "text-embedding-ada-002" + ) - # Check if input is captured - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] - input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + # Check if input is captured + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] + input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): @@ -3976,12 +2643,9 @@ async def mock_aembed_query(self, text): assert "Async query test" in input_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_embeddings_no_model_name( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test embeddings when model name is not available.""" try: @@ -3993,98 +2657,50 @@ def test_langchain_embeddings_no_model_name( integrations=[LangchainIntegration(include_prompts=False)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) - if span_streaming: - items = capture_items("span") + items = capture_items("span") - # Mock the actual API call and remove model attribute - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ): - embeddings = OpenAIEmbeddings(openai_api_key="test-key") - # Remove model attribute to test fallback - delattr(embeddings, "model") - if hasattr(embeddings, "model_name"): - delattr(embeddings, "model_name") - - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() - - with start_transaction(name="test_embeddings_no_model"): - embeddings.embed_documents(["Test"]) - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find embeddings span - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 + # Mock the actual API call and remove model attribute + with mock.patch.object( + OpenAIEmbeddings, + "embed_documents", + wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], + ): + embeddings = OpenAIEmbeddings(openai_api_key="test-key") + # Remove model attribute to test fallback + delattr(embeddings, "model") + if hasattr(embeddings, "model_name"): + delattr(embeddings, "model_name") - embeddings_span = embeddings_spans[0] - assert embeddings_span["name"] == "embeddings" - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - # Model name should not be set if not available - assert ( - "gen_ai.request.model" not in embeddings_span["attributes"] - or embeddings_span["attributes"]["gen_ai.request.model"] is None - ) - else: - events = capture_events() + # Force setup to re-run to ensure our mock is wrapped + LangchainIntegration.setup_once() - # Mock the actual API call and remove model attribute - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ): - embeddings = OpenAIEmbeddings(openai_api_key="test-key") - # Remove model attribute to test fallback - delattr(embeddings, "model") - if hasattr(embeddings, "model_name"): - delattr(embeddings, "model_name") - - # Force setup to re-run to ensure our mock is wrapped - LangchainIntegration.setup_once() - - with start_transaction(name="test_embeddings_no_model"): - embeddings.embed_documents(["Test"]) - - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - - # Find embeddings span - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 1 + embeddings.embed_documents(["Test"]) - embeddings_span = embeddings_spans[0] - assert embeddings_span["description"] == "embeddings" - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - # Model name should not be set if not available - assert ( - "gen_ai.request.model" not in embeddings_span["data"] - or embeddings_span["data"]["gen_ai.request.model"] is None - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find embeddings span + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + assert len(embeddings_spans) == 1 + + embeddings_span = embeddings_spans[0] + assert embeddings_span["name"] == "embeddings" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + # Model name should not be set if not available + assert ( + "gen_ai.request.model" not in embeddings_span["attributes"] + or embeddings_span["attributes"]["gen_ai.request.model"] is None + ) -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_embeddings_integration_disabled( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that embeddings are not traced when integration is disabled.""" try: @@ -4095,69 +2711,36 @@ def test_langchain_embeddings_integration_disabled( sentry_init( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("span") - # Initialize without LangchainIntegration - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - return_value=[[0.1, 0.2, 0.3]], - ): - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - with start_transaction(name="test_embeddings_disabled"): - embeddings.embed_documents(["Test"]) - - # Check that no embeddings spans were created - sentry_sdk.flush() - spans = [item.payload for item in items] - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - # Should be empty since integration is disabled - assert len(embeddings_spans) == 0 - else: - events = capture_events() - - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - return_value=[[0.1, 0.2, 0.3]], - ): - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) + with mock.patch.object( + OpenAIEmbeddings, + "embed_documents", + return_value=[[0.1, 0.2, 0.3]], + ): + embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" + ) - with start_transaction(name="test_embeddings_disabled"): - embeddings.embed_documents(["Test"]) + embeddings.embed_documents(["Test"]) - # Check that no embeddings spans were created - if events: - tx = events[0] - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - # Should be empty since integration is disabled - assert len(embeddings_spans) == 0 + # Check that no embeddings spans were created + sentry_sdk.flush() + spans = [item.payload for item in items] + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + # Should be empty since integration is disabled + assert len(embeddings_spans) == 0 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_embeddings_multiple_providers( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that embeddings work with different providers.""" try: @@ -4170,107 +2753,54 @@ def test_langchain_embeddings_multiple_providers( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, - ) - if span_streaming: - items = capture_items("span") - - # Mock both providers - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ), mock.patch.object( - AzureOpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.4, 0.5, 0.6] for _ in texts], - ): - openai_embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - azure_embeddings = AzureOpenAIEmbeddings( - model="text-embedding-ada-002", - azure_endpoint="https://test.openai.azure.com/", - openai_api_key="test-key", - ) - - # Force setup to re-run - LangchainIntegration.setup_once() - - with start_transaction(name="test_multiple_providers"): - openai_embeddings.embed_documents(["OpenAI test"]) - azure_embeddings.embed_documents(["Azure test"]) - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find embeddings spans - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - # Should have 2 spans, one for each provider - assert len(embeddings_spans) == 2 - - # Verify both spans have proper data - for span in embeddings_spans: - assert span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - span["attributes"]["gen_ai.request.model"] == "text-embedding-ada-002" - ) - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] - else: - events = capture_events() - - # Mock both providers - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ), mock.patch.object( - AzureOpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.4, 0.5, 0.6] for _ in texts], - ): - openai_embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - azure_embeddings = AzureOpenAIEmbeddings( - model="text-embedding-ada-002", - azure_endpoint="https://test.openai.azure.com/", - openai_api_key="test-key", - ) + trace_lifecycle="stream", + ) + items = capture_items("span") - # Force setup to re-run - LangchainIntegration.setup_once() + # Mock both providers + with mock.patch.object( + OpenAIEmbeddings, + "embed_documents", + wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], + ), mock.patch.object( + AzureOpenAIEmbeddings, + "embed_documents", + wraps=lambda self, texts: [[0.4, 0.5, 0.6] for _ in texts], + ): + openai_embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" + ) + azure_embeddings = AzureOpenAIEmbeddings( + model="text-embedding-ada-002", + azure_endpoint="https://test.openai.azure.com/", + openai_api_key="test-key", + ) - with start_transaction(name="test_multiple_providers"): - openai_embeddings.embed_documents(["OpenAI test"]) - azure_embeddings.embed_documents(["Azure test"]) + # Force setup to re-run + LangchainIntegration.setup_once() - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" + openai_embeddings.embed_documents(["OpenAI test"]) + azure_embeddings.embed_documents(["Azure test"]) - # Find embeddings spans - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - # Should have 2 spans, one for each provider - assert len(embeddings_spans) == 2 + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find embeddings spans + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + # Should have 2 spans, one for each provider + assert len(embeddings_spans) == 2 - # Verify both spans have proper data - for span in embeddings_spans: - assert span["data"]["gen_ai.operation.name"] == "embeddings" - assert span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] + # Verify both spans have proper data + for span in embeddings_spans: + assert span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert span["attributes"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] -def test_langchain_embeddings_error_handling(sentry_init, capture_events): +def test_langchain_embeddings_error_handling(sentry_init, capture_items): """Test that errors in embeddings are properly captured.""" try: from langchain_openai import OpenAIEmbeddings @@ -4282,9 +2812,9 @@ def test_langchain_embeddings_error_handling(sentry_init, capture_events): disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) - events = capture_events() + events = capture_items("event") # Mock the API call to raise an error with mock.patch.object( @@ -4299,7 +2829,7 @@ def test_langchain_embeddings_error_handling(sentry_init, capture_events): # Force setup to re-run LangchainIntegration.setup_once() - with start_transaction(name="test_embeddings_error"), pytest.raises(ValueError): + with pytest.raises(ValueError): embeddings.embed_documents(["Test"]) # The error should be captured @@ -4310,12 +2840,9 @@ def test_langchain_embeddings_error_handling(sentry_init, capture_events): # but the span should still be created -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_embeddings_multiple_calls( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that multiple embeddings calls within a transaction are all traced.""" try: @@ -4328,121 +2855,62 @@ def test_langchain_embeddings_multiple_calls( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, - ) - if span_streaming: - items = capture_items("span") - - # Mock the actual API calls - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ), mock.patch.object( - OpenAIEmbeddings, - "embed_query", - wraps=lambda self, text: [0.4, 0.5, 0.6], - ): - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run - LangchainIntegration.setup_once() - - with start_transaction(name="test_multiple_embeddings"): - # Call embed_documents - embeddings.embed_documents(["First batch", "Second batch"]) - # Call embed_query - embeddings.embed_query("Single query") - # Call embed_documents again - embeddings.embed_documents(["Third batch"]) - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find embeddings spans - should have 3 (2 embed_documents + 1 embed_query) - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 3 - - # Verify all spans have proper data - for span in embeddings_spans: - assert span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - span["attributes"]["gen_ai.request.model"] == "text-embedding-ada-002" - ) - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] - - # Verify the input data is different for each span - input_data_list = [ - span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - for span in embeddings_spans - ] - else: - events = capture_events() - - # Mock the actual API calls - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ), mock.patch.object( - OpenAIEmbeddings, - "embed_query", - wraps=lambda self, text: [0.4, 0.5, 0.6], - ): - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) + trace_lifecycle="stream", + ) + items = capture_items("span") - # Force setup to re-run - LangchainIntegration.setup_once() - - with start_transaction(name="test_multiple_embeddings"): - # Call embed_documents - embeddings.embed_documents(["First batch", "Second batch"]) - # Call embed_query - embeddings.embed_query("Single query") - # Call embed_documents again - embeddings.embed_documents(["Third batch"]) - - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - - # Find embeddings spans - should have 3 (2 embed_documents + 1 embed_query) - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 3 + # Mock the actual API calls + with mock.patch.object( + OpenAIEmbeddings, + "embed_documents", + wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], + ), mock.patch.object( + OpenAIEmbeddings, + "embed_query", + wraps=lambda self, text: [0.4, 0.5, 0.6], + ): + embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" + ) - # Verify all spans have proper data - for span in embeddings_spans: - assert span["data"]["gen_ai.operation.name"] == "embeddings" - assert span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] + # Force setup to re-run + LangchainIntegration.setup_once() - # Verify the input data is different for each span - input_data_list = [ - span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] for span in embeddings_spans - ] + # Call embed_documents + embeddings.embed_documents(["First batch", "Second batch"]) + # Call embed_query + embeddings.embed_query("Single query") + # Call embed_documents again + embeddings.embed_documents(["Third batch"]) + + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find embeddings spans - should have 3 (2 embed_documents + 1 embed_query) + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + assert len(embeddings_spans) == 3 + + # Verify all spans have proper data + for span in embeddings_spans: + assert span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert span["attributes"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] + + # Verify the input data is different for each span + input_data_list = [ + span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + for span in embeddings_spans + ] # They should all be different (different inputs) assert len(set(str(data) for data in input_data_list)) == 3 -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_embeddings_span_hierarchy( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that embeddings spans are properly nested within parent spans.""" try: @@ -4455,109 +2923,59 @@ def test_langchain_embeddings_span_hierarchy( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) - if span_streaming: - items = capture_items("span") - - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ): - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run - LangchainIntegration.setup_once() - - with sentry_sdk.traces.start_span( - name="test_span_hierarchy" - ), sentry_sdk.traces.start_span( - name="custom operation", - attributes={ - "sentry.op": "custom", - }, - ): - embeddings.embed_documents(["Test within custom span"]) - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find all spans - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - custom_spans = [ - span for span in spans if span["attributes"].get("sentry.op") == "custom" - ] - - assert len(embeddings_spans) == 1 - assert len(custom_spans) == 1 + items = capture_items("span") - # Both spans should exist - embeddings_span = embeddings_spans[0] - custom_span = custom_spans[0] + # Mock the actual API call + with mock.patch.object( + OpenAIEmbeddings, + "embed_documents", + wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], + ): + embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" + ) - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert custom_span["name"] == "custom operation" - else: - events = capture_events() + # Force setup to re-run + LangchainIntegration.setup_once() - # Mock the actual API call - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], + with sentry_sdk.traces.start_span( + name="test_span_hierarchy" + ), sentry_sdk.traces.start_span( + name="custom operation", + attributes={ + "sentry.op": "custom", + }, ): - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) + embeddings.embed_documents(["Test within custom span"]) - # Force setup to re-run - LangchainIntegration.setup_once() - - with start_transaction(name="test_span_hierarchy"), sentry_sdk.start_span( - op="custom", name="custom operation" - ): - embeddings.embed_documents(["Test within custom span"]) - - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - - # Find all spans - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - custom_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "custom" - ] + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find all spans + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + custom_spans = [ + span for span in spans if span["attributes"].get("sentry.op") == "custom" + ] - assert len(embeddings_spans) == 1 - assert len(custom_spans) == 1 + assert len(embeddings_spans) == 1 + assert len(custom_spans) == 1 - # Both spans should exist - embeddings_span = embeddings_spans[0] - custom_span = custom_spans[0] + # Both spans should exist + embeddings_span = embeddings_spans[0] + custom_span = custom_spans[0] - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert custom_span["description"] == "custom operation" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert custom_span["name"] == "custom operation" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_langchain_embeddings_with_list_and_string_inputs( sentry_init, - capture_events, capture_items, - span_streaming, ): """Test that embeddings correctly handle both list and string inputs.""" try: @@ -4570,108 +2988,52 @@ def test_langchain_embeddings_with_list_and_string_inputs( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, - ) - if span_streaming: - items = capture_items("span") - - # Mock the actual API calls - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ), mock.patch.object( - OpenAIEmbeddings, - "embed_query", - wraps=lambda self, text: [0.4, 0.5, 0.6], - ): - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) - - # Force setup to re-run - LangchainIntegration.setup_once() + trace_lifecycle="stream", + ) + items = capture_items("span") - with start_transaction(name="test_input_types"): - # embed_documents takes a list - embeddings.embed_documents( - ["List item 1", "List item 2", "List item 3"] - ) - # embed_query takes a string - embeddings.embed_query("Single string query") - - sentry_sdk.flush() - spans = [item.payload for item in items] - # Find embeddings spans - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 2 - - # Both should have input data captured as lists - for span in embeddings_spans: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] - input_data = span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - # Input should be normalized to list format - if isinstance(input_data, str): - # If serialized, should contain the input text - assert ( - "List item" in input_data or "Single string query" in input_data - ), f"Expected input text in serialized data: {input_data}" - else: - events = capture_events() - - # Mock the actual API calls - with mock.patch.object( - OpenAIEmbeddings, - "embed_documents", - wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], - ), mock.patch.object( - OpenAIEmbeddings, - "embed_query", - wraps=lambda self, text: [0.4, 0.5, 0.6], - ): - embeddings = OpenAIEmbeddings( - model="text-embedding-ada-002", openai_api_key="test-key" - ) + # Mock the actual API calls + with mock.patch.object( + OpenAIEmbeddings, + "embed_documents", + wraps=lambda self, texts: [[0.1, 0.2, 0.3] for _ in texts], + ), mock.patch.object( + OpenAIEmbeddings, + "embed_query", + wraps=lambda self, text: [0.4, 0.5, 0.6], + ): + embeddings = OpenAIEmbeddings( + model="text-embedding-ada-002", openai_api_key="test-key" + ) - # Force setup to re-run - LangchainIntegration.setup_once() + # Force setup to re-run + LangchainIntegration.setup_once() - with start_transaction(name="test_input_types"): - # embed_documents takes a list - embeddings.embed_documents( - ["List item 1", "List item 2", "List item 3"] - ) - # embed_query takes a string - embeddings.embed_query("Single string query") - - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - - # Find embeddings spans - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - assert len(embeddings_spans) == 2 + # embed_documents takes a list + embeddings.embed_documents(["List item 1", "List item 2", "List item 3"]) + # embed_query takes a string + embeddings.embed_query("Single string query") - # Both should have input data captured as lists - for span in embeddings_spans: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] - input_data = span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - # Input should be normalized to list format - if isinstance(input_data, str): - # If serialized, should contain the input text - assert ( - "List item" in input_data or "Single string query" in input_data - ), f"Expected input text in serialized data: {input_data}" + sentry_sdk.flush() + spans = [item.payload for item in items] + # Find embeddings spans + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + assert len(embeddings_spans) == 2 + + # Both should have input data captured as lists + for span in embeddings_spans: + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] + input_data = span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + # Input should be normalized to list format + if isinstance(input_data, str): + # If serialized, should contain the input text + assert "List item" in input_data or "Single string query" in input_data, ( + f"Expected input text in serialized data: {input_data}" + ) # Tests for multimodal content transformation functions @@ -4885,7 +3247,6 @@ def test_transform_google_file_data(self): } -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "ai_type,expected_system", [ @@ -4932,18 +3293,15 @@ def test_transform_google_file_data(self): ) def test_langchain_ai_system_detection( sentry_init, - capture_events, capture_items, ai_type, expected_system, - span_streaming, ): sentry_init( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) callback = SentryLangchainCallback(include_prompts=True) @@ -4951,69 +3309,34 @@ def test_langchain_ai_system_detection( run_id = "test-ai-system-uuid" serialized = {"_type": ai_type} if ai_type is not None else {} prompts = ["Test prompt"] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(): - callback.on_llm_start( - serialized=serialized, - prompts=prompts, - run_id=run_id, - invocation_params={"_type": ai_type, "model": "test-model"}, - ) + callback.on_llm_start( + serialized=serialized, + prompts=prompts, + run_id=run_id, + invocation_params={"_type": ai_type, "model": "test-model"}, + ) - generation = Mock(text="Test response", message=None) - response = Mock(generations=[[generation]]) - callback.on_llm_end(response=response, run_id=run_id) + generation = Mock(text="Test response", message=None) + response = Mock(generations=[[generation]]) + callback.on_llm_end(response=response, run_id=run_id) - sentry_sdk.flush() - spans = [item.payload for item in items] - llm_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.text_completion" - ] + sentry_sdk.flush() + spans = [item.payload for item in items] + llm_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.text_completion" + ] - assert len(llm_spans) > 0 - llm_span = llm_spans[0] + assert len(llm_spans) > 0 + llm_span = llm_spans[0] - if expected_system is not None: - assert llm_span["attributes"][SPANDATA.GEN_AI_SYSTEM] == expected_system - else: - assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("attributes", {}) + if expected_system is not None: + assert llm_span["attributes"][SPANDATA.GEN_AI_SYSTEM] == expected_system else: - events = capture_events() - - with start_transaction(): - callback.on_llm_start( - serialized=serialized, - prompts=prompts, - run_id=run_id, - invocation_params={"_type": ai_type, "model": "test-model"}, - ) - - generation = Mock(text="Test response", message=None) - response = Mock(generations=[[generation]]) - callback.on_llm_end(response=response, run_id=run_id) - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - llm_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" - ] - - assert len(llm_spans) > 0 - llm_span = llm_spans[0] - - if expected_system is not None: - assert llm_span["data"][SPANDATA.GEN_AI_SYSTEM] == expected_system - else: - assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("data", {}) + assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("attributes", {}) class TestTransformLangchainMessageContent: @@ -5132,7 +3455,6 @@ def test_transform_list_with_legacy_image_url(self): } -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", [ @@ -5247,15 +3569,13 @@ def test_langchain_chat_data_collection( include_prompts, expected_present, expected_absent, - span_streaming, ): sentry_init_kwargs = dict( integrations=[LangchainIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -5289,14 +3609,13 @@ def test_langchain_chat_data_collection( openai_api_key="badkey", ) - streamed = span_streaming - captured = capture_items("span") if streamed else capture_events() + captured = capture_items("span") with patch.object( llm.client._client._client, "send", return_value=model_response, - ) as _, start_transaction(): + ): llm.invoke( [ SystemMessage(content="You are a helpful assistant."), @@ -5304,17 +3623,10 @@ def test_langchain_chat_data_collection( ] ) - if streamed: - sentry_sdk.flush() - spans = [item.payload for item in captured] - (span,) = [ - s for s in spans if s["attributes"].get("sentry.op") == "gen_ai.chat" - ] - span_data = span["attributes"] - else: - (event,) = captured - (span,) = [s for s in event["spans"] if s["op"] == "gen_ai.chat"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in captured] + (span,) = [s for s in spans if s["attributes"].get("sentry.op") == "gen_ai.chat"] + span_data = span["attributes"] for key, expected in expected_present.items(): assert key in span_data, f"{key} should have been collected" @@ -5332,7 +3644,6 @@ def test_langchain_chat_data_collection( assert span_data[SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", [ @@ -5453,15 +3764,13 @@ def test_langchain_text_completion_data_collection( include_prompts, expected_present, expected_absent, - span_streaming, ): sentry_init_kwargs = dict( integrations=[LangchainIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -5497,29 +3806,21 @@ def test_langchain_text_completion_data_collection( openai_api_key="badkey", ) - streamed = span_streaming - captured = capture_items("span") if streamed else capture_events() + captured = capture_items("span") with patch.object( model.client._client._client, "send", return_value=model_response, - ) as _, start_transaction(): + ): model.invoke("What is the capital of France?") - if streamed: - sentry_sdk.flush() - spans = [item.payload for item in captured] - (span,) = [ - s - for s in spans - if s["attributes"].get("sentry.op") == "gen_ai.text_completion" - ] - span_data = span["attributes"] - else: - (event,) = captured - (span,) = [s for s in event["spans"] if s["op"] == "gen_ai.text_completion"] - span_data = span["data"] + sentry_sdk.flush() + spans = [item.payload for item in captured] + (span,) = [ + s for s in spans if s["attributes"].get("sentry.op") == "gen_ai.text_completion" + ] + span_data = span["attributes"] for key, expected in expected_present.items(): assert key in span_data, f"{key} should have been collected" @@ -5538,7 +3839,6 @@ def test_langchain_text_completion_data_collection( assert span_data[SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 25 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", [ @@ -5635,15 +3935,13 @@ def test_langchain_data_collection_tools( include_prompts, expected_present, expected_absent, - span_streaming, ): sentry_init_kwargs = dict( integrations=[LangchainIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -5678,35 +3976,27 @@ def test_langchain_data_collection_tools( agent = create_openai_tools_agent(llm, [get_word_length], prompt) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) - streamed = span_streaming - captured = capture_items("span") if streamed else capture_events() + captured = capture_items("span") with patch.object( llm.client._client._client, "send", side_effect=[tool_response, final_response], - ) as _, start_transaction(): + ): agent_executor.invoke({"input": "How many letters in the word eudca"}) - if streamed: - sentry_sdk.flush() - spans = [item.payload for item in captured] - chat_spans = [ - s["attributes"] - for s in spans - if s["attributes"].get("sentry.op") == "gen_ai.chat" - ] - invoke_agent_span = next( - s["attributes"] - for s in spans - if s["attributes"].get("sentry.op") == "gen_ai.invoke_agent" - ) - else: - (event,) = captured - chat_spans = [s["data"] for s in event["spans"] if s["op"] == "gen_ai.chat"] - invoke_agent_span = next( - s["data"] for s in event["spans"] if s["op"] == "gen_ai.invoke_agent" - ) + sentry_sdk.flush() + spans = [item.payload for item in captured] + chat_spans = [ + s["attributes"] + for s in spans + if s["attributes"].get("sentry.op") == "gen_ai.chat" + ] + invoke_agent_span = next( + s["attributes"] + for s in spans + if s["attributes"].get("sentry.op") == "gen_ai.invoke_agent" + ) assert len(chat_spans) == 2 @@ -5728,7 +4018,6 @@ def test_langchain_data_collection_tools( assert (key in invoke_agent_span) is collected -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("send_default_pii", [True, False]) @pytest.mark.parametrize( "data_collection,tool_calls_collected", @@ -5762,15 +4051,13 @@ def test_langchain_data_collection_request_tool_call_params( data_collection, send_default_pii, tool_calls_collected, - span_streaming, ): sentry_init_kwargs = dict( integrations=[LangchainIntegration(include_prompts=False)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -5779,35 +4066,29 @@ def test_langchain_data_collection_request_tool_call_params( callback = SentryLangchainCallback(include_prompts=False) - streamed = span_streaming - captured = capture_items("span") if streamed else capture_events() - - with start_transaction(): - callback.on_chat_model_start( - serialized={}, - messages=[[HumanMessage(content="How many letters in the word eudca")]], - run_id="test-request-tool-calls-uuid", - invocation_params={ - "model": "gpt-3.5-turbo", - "function_call": {"name": "get_word_length"}, - }, - ) - callback.on_llm_end( - response=LLMResult(generations=[[]]), - run_id="test-request-tool-calls-uuid", - ) + captured = capture_items("span") - if streamed: - sentry_sdk.flush() - spans = [item.payload for item in captured] - (span_data,) = [ - s["attributes"] - for s in spans - if s["attributes"].get("sentry.op") == "gen_ai.chat" - ] - else: - (event,) = captured - (span_data,) = [s["data"] for s in event["spans"] if s["op"] == "gen_ai.chat"] + callback.on_chat_model_start( + serialized={}, + messages=[[HumanMessage(content="How many letters in the word eudca")]], + run_id="test-request-tool-calls-uuid", + invocation_params={ + "model": "gpt-3.5-turbo", + "function_call": {"name": "get_word_length"}, + }, + ) + callback.on_llm_end( + response=LLMResult(generations=[[]]), + run_id="test-request-tool-calls-uuid", + ) + + sentry_sdk.flush() + spans = [item.payload for item in captured] + (span_data,) = [ + s["attributes"] + for s in spans + if s["attributes"].get("sentry.op") == "gen_ai.chat" + ] if tool_calls_collected: assert "get_word_length" in span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] @@ -5818,7 +4099,6 @@ def test_langchain_data_collection_request_tool_call_params( assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", [ @@ -5901,15 +4181,13 @@ def test_langchain_tool_execution_data_collection( include_prompts, expected_present, expected_absent, - span_streaming, ): sentry_init_kwargs = dict( integrations=[LangchainIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -5944,29 +4222,22 @@ def test_langchain_tool_execution_data_collection( agent = create_openai_tools_agent(llm, [get_word_length], prompt) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) - streamed = span_streaming - captured = capture_items("span") if streamed else capture_events() + captured = capture_items("span") with patch.object( llm.client._client._client, "send", side_effect=[tool_response, final_response], - ) as _, start_transaction(): + ): agent_executor.invoke({"input": "How many letters in the word eudca"}) - if streamed: - sentry_sdk.flush() - spans = [item.payload for item in captured] - span_data = next( - s["attributes"] - for s in spans - if s["attributes"].get("sentry.op") == "gen_ai.execute_tool" - ) - else: - (event,) = captured - span_data = next( - s["data"] for s in event["spans"] if s["op"] == "gen_ai.execute_tool" - ) + sentry_sdk.flush() + spans = [item.payload for item in captured] + span_data = next( + s["attributes"] + for s in spans + if s["attributes"].get("sentry.op") == "gen_ai.execute_tool" + ) for key, expected in expected_present.items(): assert key in span_data, f"{key} should have been collected" @@ -5983,7 +4254,6 @@ def test_langchain_tool_execution_data_collection( assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "execute_tool" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("agent_method", ["invoke", "stream"]) @pytest.mark.parametrize( "data_collection,send_default_pii,include_prompts,expected_present,expected_absent", @@ -6074,15 +4344,13 @@ def test_langchain_agent_executor_data_collection( expected_present, expected_absent, agent_method, - span_streaming, ): sentry_init_kwargs = dict( integrations=[LangchainIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -6117,32 +4385,25 @@ def test_langchain_agent_executor_data_collection( agent = create_openai_tools_agent(llm, [get_word_length], prompt) agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) - streamed = span_streaming - captured = capture_items("span") if streamed else capture_events() + captured = capture_items("span") with patch.object( llm.client._client._client, "send", side_effect=[tool_response, final_response], - ) as _, start_transaction(): + ): if agent_method == "invoke": agent_executor.invoke({"input": "How many letters in the word eudca"}) else: list(agent_executor.stream({"input": "How many letters in the word eudca"})) - if streamed: - sentry_sdk.flush() - spans = [item.payload for item in captured] - span_data = next( - s["attributes"] - for s in spans - if s["attributes"].get("sentry.op") == "gen_ai.invoke_agent" - ) - else: - (event,) = captured - span_data = next( - s["data"] for s in event["spans"] if s["op"] == "gen_ai.invoke_agent" - ) + sentry_sdk.flush() + spans = [item.payload for item in captured] + span_data = next( + s["attributes"] + for s in spans + if s["attributes"].get("sentry.op") == "gen_ai.invoke_agent" + ) for key, expected in expected_present.items(): assert key in span_data, f"{key} should have been collected" @@ -6159,7 +4420,6 @@ def test_langchain_agent_executor_data_collection( assert span_data[SPANDATA.GEN_AI_RESPONSE_STREAMING] is (agent_method == "stream") -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "method", ["embed_documents", "embed_query", "aembed_documents", "aembed_query"], @@ -6228,7 +4488,6 @@ async def test_langchain_embeddings_data_collection( include_prompts, inputs_collected, method, - span_streaming, ): try: from langchain_openai import OpenAIEmbeddings @@ -6240,8 +4499,7 @@ async def test_langchain_embeddings_data_collection( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) if data_collection is not None: sentry_init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -6269,8 +4527,7 @@ async def mock_aembed_query(self, text): is_query = method.endswith("embed_query") embeddings_input = "Hello world" if is_query else ["Hello world", "Test document"] - streamed = span_streaming - captured = capture_items("span") if streamed else capture_events() + captured = capture_items("span") with mock.patch.object(OpenAIEmbeddings, method, wraps=mocks[method]): embeddings = OpenAIEmbeddings( @@ -6280,25 +4537,18 @@ async def mock_aembed_query(self, text): # Force setup to re-run to ensure our mock is wrapped LangchainIntegration.setup_once() - with start_transaction(name="test_embeddings_data_collection"): - if method.startswith("a"): - await getattr(embeddings, method)(embeddings_input) - else: - getattr(embeddings, method)(embeddings_input) - - if streamed: - sentry_sdk.flush() - spans = [item.payload for item in captured] - span_data = next( - s["attributes"] - for s in spans - if s["attributes"].get("sentry.op") == "gen_ai.embeddings" - ) - else: - (event,) = captured - span_data = next( - s["data"] for s in event["spans"] if s["op"] == "gen_ai.embeddings" - ) + if method.startswith("a"): + await getattr(embeddings, method)(embeddings_input) + else: + getattr(embeddings, method)(embeddings_input) + + sentry_sdk.flush() + spans = [item.payload for item in captured] + span_data = next( + s["attributes"] + for s in spans + if s["attributes"].get("sentry.op") == "gen_ai.embeddings" + ) if inputs_collected: assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span_data, (