From b0c4169d43a9556779da32f7687155fd0ce9f66f Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 3 Sep 2026 13:55:54 +0200 Subject: [PATCH 1/5] ref(openai): Remove transaction-based tracing --- sentry_sdk/integrations/openai.py | 349 +- tests/integrations/openai/test_openai.py | 4554 +++++++--------------- 2 files changed, 1425 insertions(+), 3478 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 5ed2e9172d..b90a29c3fa 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -36,7 +36,6 @@ ) from sentry_sdk.ai.monitoring import record_token_usage from sentry_sdk.ai.utils import ( - get_start_span_function, normalize_message_roles, set_data_normalized, truncate_and_annotate_embedding_inputs, @@ -80,7 +79,6 @@ from openai.types.responses.response_usage import ResponseUsage from sentry_sdk._types import TextPart - from sentry_sdk.tracing import Span try: try: @@ -181,7 +179,7 @@ def _has_attr_and_is_int( def _calculate_completions_token_usage( messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]", response: "Any", - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", streaming_message_responses: "Optional[List[str]]", streaming_message_total_token_usage: "Optional[CompletionUsage]", count_tokens: "Callable[..., Any]", @@ -263,7 +261,7 @@ def _calculate_completions_token_usage( def _calculate_responses_token_usage( input: "Any", response: "Any", - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", streaming_message_responses: "Optional[List[str]]", count_tokens: "Callable[..., Any]", ) -> None: @@ -339,31 +337,27 @@ def _calculate_responses_token_usage( def _set_responses_api_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") model = kwargs.get("model") if model is not None: - set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model) max_tokens = kwargs.get("max_output_tokens") if max_tokens is not None and _is_given(max_tokens): - set_on_span(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) temperature = kwargs.get("temperature") if temperature is not None and _is_given(temperature): - set_on_span(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) top_p = kwargs.get("top_p") if top_p is not None and _is_given(top_p): - set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) conversation = kwargs.get("conversation") if conversation is not None and _is_given(conversation): @@ -373,11 +367,11 @@ def _set_responses_api_input_data( elif isinstance(conversation, dict): conversation_id = conversation.get("id") if conversation_id is not None: - set_on_span(SPANDATA.GEN_AI_CONVERSATION_ID, conversation_id) + span.set_attribute(SPANDATA.GEN_AI_CONVERSATION_ID, conversation_id) reasoning = kwargs.get("reasoning") if isinstance(reasoning, dict) and "effort" in reasoning: - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL, reasoning["effort"], ) @@ -387,7 +381,7 @@ def _set_responses_api_input_data( if client_options["data_collection"]["gen_ai"]["inputs"]: tools = kwargs.get("tools") if tools is not None and _is_given(tools): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions_responses(tools)), ) @@ -398,7 +392,7 @@ def _set_responses_api_input_data( # line below tools = kwargs.get("tools") if tools is not None and _is_given(tools): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions_responses(tools)), ) @@ -418,7 +412,7 @@ def _set_responses_api_input_data( if messages is None: if has_explicit_instructions: - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps( [ @@ -445,7 +439,7 @@ def _set_responses_api_input_data( system_instructions ) if len(instructions_text_parts) > 0: - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps(instructions_text_parts), ) @@ -486,50 +480,46 @@ def _set_responses_api_input_data( def _set_completions_api_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") model = kwargs.get("model") if model is not None: - set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model) max_tokens = kwargs.get("max_tokens") if max_tokens is not None and _is_given(max_tokens): - set_on_span(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) presence_penalty = kwargs.get("presence_penalty") if presence_penalty is not None and _is_given(presence_penalty): - set_on_span(SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, presence_penalty) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, presence_penalty) frequency_penalty = kwargs.get("frequency_penalty") if frequency_penalty is not None and _is_given(frequency_penalty): - set_on_span(SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, frequency_penalty) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, frequency_penalty) temperature = kwargs.get("temperature") if temperature is not None and _is_given(temperature): - set_on_span(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) top_p = kwargs.get("top_p") if top_p is not None and _is_given(top_p): - set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) reasoning_level = kwargs.get("reasoning_effort") if reasoning_level is not None and _is_given(reasoning_level): - set_on_span(SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL, reasoning_level) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL, reasoning_level) client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): if client.options["data_collection"]["gen_ai"]["inputs"]: tools = kwargs.get("tools") if tools is not None and _is_given(tools): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions_completions(tools)), ) @@ -540,7 +530,7 @@ def _set_completions_api_input_data( # line below tools = kwargs.get("tools") if tools is not None and _is_given(tools): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions_completions(tools)), ) @@ -581,7 +571,7 @@ def _set_completions_api_input_data( system_instructions = _get_system_instructions_completions(messages) if len(system_instructions) > 0: - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps(_transform_system_instructions_completions(system_instructions)), ) @@ -607,20 +597,16 @@ def _set_completions_api_input_data( def _set_embeddings_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) - model = kwargs.get("model") if model is not None: - set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model) messages: "Optional[Union[str, SequenceNotStr[str], Iterable[int], Iterable[Iterable[int]]]]" = kwargs.get( "input" @@ -673,7 +659,7 @@ def _set_embeddings_input_data( def _set_common_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", response: "Union[ChatCompletion, Stream[ChatCompletionChunk], AsyncStream[ChatCompletionChunk], Response, Stream[ResponseStreamEvent], AsyncStream[ResponseStreamEvent], CreateEmbeddingResponse]", input: "Any", integration: "OpenAIIntegration", @@ -836,27 +822,15 @@ def _new_sync_chat_completion( # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/completions.py#L585 is_streaming_response = kwargs.get("stream", False) or False - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"chat {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_CHAT, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, - }, - ) - - else: - span = get_start_span_function()( - op=consts.OP.GEN_AI_CHAT, - name=f"chat {model}", - origin=OpenAIIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, is_streaming_response) + span = sentry_sdk.traces.start_span( + name=f"chat {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_CHAT, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, + }, + ) _set_completions_api_input_data(span, kwargs, integration) @@ -923,26 +897,15 @@ async def _new_async_chat_completion( # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/completions.py#L585 is_streaming_response = kwargs.get("stream", False) or False - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"chat {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_CHAT, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, - }, - ) - else: - span = get_start_span_function()( - op=consts.OP.GEN_AI_CHAT, - name=f"chat {model}", - origin=OpenAIIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, is_streaming_response) + span = sentry_sdk.traces.start_span( + name=f"chat {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_CHAT, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, + }, + ) _set_completions_api_input_data(span, kwargs, integration) @@ -982,7 +945,7 @@ async def _new_async_chat_completion( def _set_completions_api_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", response: "Union[ChatCompletion, Stream[ChatCompletionChunk], AsyncStream[ChatCompletionChunk]]", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", @@ -1003,7 +966,7 @@ def _set_completions_api_output_data( def _wrap_synchronous_completions_chunk_iterator( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "OpenAIIntegration", start_time: "Optional[float]", messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]", @@ -1022,10 +985,7 @@ def _wrap_synchronous_completions_chunk_iterator( client = sentry_sdk.get_client() for x in old_iterator: - if isinstance(span, StreamedSpan): - span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) with capture_internal_exceptions(): if hasattr(x, "choices") and x.choices is not None: @@ -1074,7 +1034,7 @@ def _wrap_synchronous_completions_chunk_iterator( async def _wrap_asynchronous_completions_chunk_iterator( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "OpenAIIntegration", start_time: "Optional[float]", messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]", @@ -1093,10 +1053,7 @@ async def _wrap_asynchronous_completions_chunk_iterator( client = sentry_sdk.get_client() async for x in old_iterator: - if isinstance(span, StreamedSpan): - span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) with capture_internal_exceptions(): if hasattr(x, "choices") and x.choices is not None: @@ -1145,7 +1102,7 @@ async def _wrap_asynchronous_completions_chunk_iterator( def _wrap_synchronous_responses_event_iterator( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "OpenAIIntegration", start_time: "Optional[float]", input: "Optional[Union[str, list[str], ResponseInputParam]]", @@ -1173,10 +1130,7 @@ def _wrap_synchronous_responses_event_iterator( data_buf[0].append(x.delta or "") elif isinstance(x, ResponseCompletedEvent): - if isinstance(span, StreamedSpan): - span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) _calculate_responses_token_usage( input=input, @@ -1218,7 +1172,7 @@ def _wrap_synchronous_responses_event_iterator( async def _wrap_asynchronous_responses_event_iterator( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "OpenAIIntegration", start_time: "Optional[float]", input: "Optional[Union[str, list[str], ResponseInputParam]]", @@ -1246,10 +1200,7 @@ async def _wrap_asynchronous_responses_event_iterator( data_buf[0].append(x.delta or "") elif isinstance(x, ResponseCompletedEvent): - if isinstance(span, StreamedSpan): - span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) _calculate_responses_token_usage( input=input, @@ -1291,7 +1242,7 @@ async def _wrap_asynchronous_responses_event_iterator( def _set_responses_api_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", response: "Union[Response, Stream[ResponseStreamEvent], AsyncStream[ResponseStreamEvent]]", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", @@ -1312,7 +1263,7 @@ def _set_responses_api_output_data( def _set_embeddings_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", response: "CreateEmbeddingResponse", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", @@ -1372,52 +1323,29 @@ def _new_sync_embeddings_create(f: "Any", *args: "Any", **kwargs: "Any") -> "Any model = kwargs.get("model") - if has_span_streaming_enabled(client.options): - with sentry_sdk.traces.start_span( - name=f"embeddings {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_EMBEDDINGS, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - }, - ) as span: - _set_embeddings_input_data(span, kwargs, integration) - - try: - response = f(*args, **kwargs) - except Exception as exc: - exc_info = sys.exc_info() - with capture_internal_exceptions(): - _capture_exception(exc) - reraise(*exc_info) - - _set_embeddings_output_data( - span, response, kwargs, integration, finish_span=False - ) + with sentry_sdk.traces.start_span( + name=f"embeddings {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_EMBEDDINGS, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + }, + ) as span: + _set_embeddings_input_data(span, kwargs, integration) - return response - else: - with get_start_span_function()( - op=consts.OP.GEN_AI_EMBEDDINGS, - name=f"embeddings {model}", - origin=OpenAIIntegration.origin, - ) as span: - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - _set_embeddings_input_data(span, kwargs, integration) - - try: - response = f(*args, **kwargs) - except Exception as exc: - exc_info = sys.exc_info() - with capture_internal_exceptions(): - _capture_exception(exc) - reraise(*exc_info) - - _set_embeddings_output_data( - span, response, kwargs, integration, finish_span=False - ) + try: + response = f(*args, **kwargs) + except Exception as exc: + exc_info = sys.exc_info() + with capture_internal_exceptions(): + _capture_exception(exc) + reraise(*exc_info) + + _set_embeddings_output_data( + span, response, kwargs, integration, finish_span=False + ) - return response + return response async def _new_async_embeddings_create( @@ -1432,52 +1360,29 @@ async def _new_async_embeddings_create( model = kwargs.get("model") - if has_span_streaming_enabled(client.options): - with sentry_sdk.traces.start_span( - name=f"embeddings {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_EMBEDDINGS, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - }, - ) as span: - _set_embeddings_input_data(span, kwargs, integration) - - try: - response = await f(*args, **kwargs) - except Exception as exc: - exc_info = sys.exc_info() - with capture_internal_exceptions(): - _capture_exception(exc) - reraise(*exc_info) - - _set_embeddings_output_data( - span, response, kwargs, integration, finish_span=False - ) + with sentry_sdk.traces.start_span( + name=f"embeddings {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_EMBEDDINGS, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + }, + ) as span: + _set_embeddings_input_data(span, kwargs, integration) - return response - else: - with get_start_span_function()( - op=consts.OP.GEN_AI_EMBEDDINGS, - name=f"embeddings {model}", - origin=OpenAIIntegration.origin, - ) as span: - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - _set_embeddings_input_data(span, kwargs, integration) - - try: - response = await f(*args, **kwargs) - except Exception as exc: - exc_info = sys.exc_info() - with capture_internal_exceptions(): - _capture_exception(exc) - reraise(*exc_info) - - _set_embeddings_output_data( - span, response, kwargs, integration, finish_span=False - ) + try: + response = await f(*args, **kwargs) + except Exception as exc: + exc_info = sys.exc_info() + with capture_internal_exceptions(): + _capture_exception(exc) + reraise(*exc_info) + + _set_embeddings_output_data( + span, response, kwargs, integration, finish_span=False + ) - return response + return response def _wrap_embeddings_create(f: "Any") -> "Any": @@ -1519,26 +1424,15 @@ def _new_sync_responses_create( # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 is_streaming_response = kwargs.get("stream", False) or False - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"responses {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_RESPONSES, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, - }, - ) - else: - span = get_start_span_function()( - op=consts.OP.GEN_AI_RESPONSES, - name=f"responses {model}", - origin=OpenAIIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, is_streaming_response) + span = sentry_sdk.traces.start_span( + name=f"responses {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_RESPONSES, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, + }, + ) _set_responses_api_input_data(span, kwargs, integration) @@ -1595,26 +1489,15 @@ async def _new_async_responses_create( # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 is_streaming_response = kwargs.get("stream", False) or False - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"responses {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_RESPONSES, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, - }, - ) - else: - span = get_start_span_function()( - op=consts.OP.GEN_AI_RESPONSES, - name=f"responses {model}", - origin=OpenAIIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, is_streaming_response) + span = sentry_sdk.traces.start_span( + name=f"responses {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_RESPONSES, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, + }, + ) _set_responses_api_input_data(span, kwargs, integration) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 2701c88c15..e2700ea576 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -168,20 +168,16 @@ async def __call__(self, *args, **kwargs): OPENAI_VERSION <= (2, 10, 0), reason="ChatCompletionCustomToolParam is unavailable before.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) def test_chat_completion_tool_definitions( sentry_init, - capture_events, capture_items, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -198,134 +194,69 @@ def test_chat_completion_tool_definitions( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - tools=[ - ChatCompletionFunctionToolParam( - type="function", - function=FunctionDefinition( - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, - }, - strict=True, - ), - ), - ChatCompletionCustomToolParam( - type="custom", - custom=Custom( - name="name", - description="description", - ), - ), - ], - ) - - sentry_sdk.flush() - span = next(item.payload for item in items) - - assert json.loads(span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ - { - "type": "function", - "name": "name", - "description": "description", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, - }, - }, - { - "type": "custom", - "name": "name", - "description": "description", - }, - ] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - tools=[ - ChatCompletionFunctionToolParam( - type="function", - function=FunctionDefinition( - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + tools=[ + ChatCompletionFunctionToolParam( + type="function", + function=FunctionDefinition( + name="name", + description="description", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, }, - strict=True, - ), + "required": ["city", "state"], + "additionalProperties": False, + }, + strict=True, ), - ChatCompletionCustomToolParam( - type="custom", - custom=Custom( - name="name", - description="description", - ), + ), + ChatCompletionCustomToolParam( + type="custom", + custom=Custom( + name="name", + description="description", ), - ], - ) + ), + ], + ) - tx = events[0] - assert tx["type"] == "transaction" + sentry_sdk.flush() + span = next(item.payload for item in items) - assert json.loads(tx["spans"][0]["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ - { - "type": "function", - "name": "name", - "description": "description", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, + assert json.loads(span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + { + "type": "function", + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, }, + "required": ["city", "state"], + "additionalProperties": False, }, - { - "type": "custom", - "name": "name", - "description": "description", - }, - ] + }, + { + "type": "custom", + "name": "name", + "description": "description", + }, + ] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -336,20 +267,17 @@ def test_chat_completion_tool_definitions( ) def test_nonstreaming_chat_completion_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -366,95 +294,49 @@ def test_nonstreaming_chat_completion_no_prompts( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content - ) - - assert response == "the model response" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content + with start_transaction(name="openai tx"): + response = ( + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) + .choices[0] + .message.content + ) - assert response == "the model response" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert response == "the model response" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_messages,expected_system_instructions", [ @@ -535,20 +417,17 @@ def test_nonstreaming_chat_completion_no_prompts( ) def test_nonstreaming_chat_completion( sentry_init, - capture_events, capture_items, get_messages, expected_system_instructions, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -565,105 +444,58 @@ def test_nonstreaming_chat_completion( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content + with start_transaction(name="openai tx"): + response = ( + client.chat.completions.create( + model="some-model", + messages=get_messages(), + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) - - assert response == "the model response" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions - ) - - assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert ( - "Message demonstrating the absence of truncation." - in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + .choices[0] + .message.content ) - assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content - ) - - assert response == "the model response" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions - ) + assert response == "the model response" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + == expected_system_instructions + ) - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the tools parameter.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,expected_present,expected_absent", [ @@ -709,21 +541,18 @@ def test_nonstreaming_chat_completion( ) def test_completions_api_data_collection( sentry_init, - capture_events, capture_items, data_collection, expected_present, expected_absent, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -754,25 +583,14 @@ def test_completions_api_data_collection( "top_p": 0.9, "tools": EXAMPLE_COMPLETIONS_TOOLS, } + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create(**create_kwargs) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create(**create_kwargs) + with start_transaction(name="openai tx"): + client.chat.completions.create(**create_kwargs) - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" @@ -790,7 +608,6 @@ def test_completions_api_data_collection( assert key not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,expect_output", [ @@ -834,21 +651,18 @@ def test_completions_api_data_collection( ) def test_completions_api_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, expect_output, nonstreaming_chat_completions_model_response, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } if data_collection is not None: init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -869,31 +683,17 @@ def test_completions_api_data_collection_outputs( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" @@ -903,7 +703,6 @@ def test_completions_api_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,expect_output", @@ -948,21 +747,18 @@ def test_completions_api_data_collection_outputs( ) async def test_completions_api_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, expect_output, nonstreaming_chat_completions_model_response, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } if data_collection is not None: init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -983,31 +779,17 @@ async def test_completions_api_data_collection_outputs_async( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + with start_transaction(name="openai tx"): + await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" @@ -1017,20 +799,16 @@ async def test_completions_api_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_completions_api_data_collection_outputs_empty_choices( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, _experiments={"data_collection": {"gen_ai": {"outputs": True}}}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1048,37 +826,22 @@ def test_completions_api_data_collection_outputs_empty_choices( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] # No choices means no output data, even with outputs collection enabled assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,expect_output", [ @@ -1101,13 +864,11 @@ def test_completions_api_data_collection_outputs_empty_choices( ) def test_streaming_chat_completion_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, expect_output, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], @@ -1115,8 +876,7 @@ def test_streaming_chat_completion_data_collection_outputs( traces_sample_rate=1.0, send_default_pii=False, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1140,49 +900,26 @@ def test_streaming_chat_completion_data_collection_outputs( include_event_type=False, ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + ) + response_string = "".join( + map(lambda x: x.choices[0].delta.content, response_stream) + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - - assert response_string == "hello" - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - - assert response_string == "hello" - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + assert response_string == "hello" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] if expect_output: assert "hello" in span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -1190,7 +927,6 @@ def test_streaming_chat_completion_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,expect_output", @@ -1214,14 +950,12 @@ def test_streaming_chat_completion_data_collection_outputs( ) async def test_streaming_chat_completion_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, expect_output, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], @@ -1229,8 +963,7 @@ async def test_streaming_chat_completion_data_collection_outputs_async( traces_sample_rate=1.0, send_default_pii=False, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -1256,49 +989,26 @@ async def test_streaming_chat_completion_data_collection_outputs_async( ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + ) + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - ) - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello" - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - ) - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello" - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + assert response_string == "hello" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] if expect_output: assert "hello" in span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -1306,7 +1016,6 @@ async def test_streaming_chat_completion_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -1318,20 +1027,17 @@ async def test_streaming_chat_completion_data_collection_outputs_async( ) async def test_nonstreaming_chat_completion_async_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -1348,89 +1054,46 @@ async def test_nonstreaming_chat_completion_async_no_prompts( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content - - assert response == "the model response" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content + with start_transaction(name="openai tx"): + response = await client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) + response = response.choices[0].message.content - assert response == "the model response" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert response == "the model response" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "get_messages,expected_system_instructions", @@ -1512,20 +1175,17 @@ async def test_nonstreaming_chat_completion_async_no_prompts( ) async def test_nonstreaming_chat_completion_async( sentry_init, - capture_events, capture_items, get_messages, expected_system_instructions, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -1542,92 +1202,49 @@ async def test_nonstreaming_chat_completion_async( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content - - assert response == "the model response" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions + with start_transaction(name="openai tx"): + response = await client.chat.completions.create( + model="some-model", + messages=get_messages(), + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) + response = response.choices[0].message.content + + assert response == "the model response" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + == expected_system_instructions + ) - assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert ( - "Message demonstrating the absence of truncation." - in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content - - assert response == "the model response" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions - ) - - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 def tiktoken_encoding_if_installed(): @@ -1640,7 +1257,6 @@ def tiktoken_encoding_if_installed(): # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -1651,13 +1267,11 @@ def tiktoken_encoding_if_installed(): ) def test_streaming_chat_completion_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -1669,8 +1283,7 @@ def test_streaming_chat_completion_no_prompts( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1720,128 +1333,69 @@ def test_streaming_chat_completion_no_prompts( include_event_type=False, ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + stream=True, + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) + response_string = "".join( + map(lambda x: x.choices[0].delta.content, response_stream) + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - - assert response_string == "hello world" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - - assert response_string == "hello world" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert response_string == "hello world" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 7 - assert span["data"]["gen_ai.usage.total_tokens"] == 9 - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the stream_options parameter.", ) def test_streaming_chat_completion_with_usage_in_stream( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """When stream_options=include_usage is set, token usage comes from the final chunk's usage field.""" sentry_init( @@ -1849,8 +1403,7 @@ def test_streaming_chat_completion_with_usage_in_stream( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1892,68 +1445,39 @@ def test_streaming_chat_completion_with_usage_in_stream( include_event_type=False, ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - for _ in response_stream: - pass - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the stream_options parameter.", ) def test_streaming_chat_completion_empty_content_preserves_token_usage( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """Token usage from the stream is recorded even when no content is produced (e.g. content filter).""" sentry_init( @@ -1961,8 +1485,7 @@ def test_streaming_chat_completion_empty_content_preserves_token_usage( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1985,57 +1508,30 @@ def test_streaming_chat_completion_empty_content_preserves_token_usage( include_event_type=False, ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert "gen_ai.usage.output_tokens" not in span["attributes"] - assert span["attributes"]["gen_ai.usage.total_tokens"] == 20 - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - for _ in response_stream: - pass - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert "gen_ai.usage.output_tokens" not in span["data"] - assert span["data"]["gen_ai.usage.total_tokens"] == 20 + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert "gen_ai.usage.output_tokens" not in span["attributes"] + assert span["attributes"]["gen_ai.usage.total_tokens"] == 20 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the stream_options parameter.", @@ -2043,12 +1539,10 @@ def test_streaming_chat_completion_empty_content_preserves_token_usage( @pytest.mark.asyncio async def test_streaming_chat_completion_empty_content_preserves_token_usage_async( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """Token usage from the stream is recorded even when no content is produced - async variant.""" sentry_init( @@ -2056,8 +1550,7 @@ async def test_streaming_chat_completion_empty_content_preserves_token_usage_asy disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -2082,57 +1575,30 @@ async def test_streaming_chat_completion_empty_content_preserves_token_usage_asy ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + async for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - async for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert "gen_ai.usage.output_tokens" not in span["attributes"] - assert span["attributes"]["gen_ai.usage.total_tokens"] == 20 - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - async for _ in response_stream: - pass - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert "gen_ai.usage.output_tokens" not in span["data"] - assert span["data"]["gen_ai.usage.total_tokens"] == 20 + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert "gen_ai.usage.output_tokens" not in span["attributes"] + assert span["attributes"]["gen_ai.usage.total_tokens"] == 20 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the stream_options parameter.", @@ -2140,12 +1606,10 @@ async def test_streaming_chat_completion_empty_content_preserves_token_usage_asy @pytest.mark.asyncio async def test_streaming_chat_completion_async_with_usage_in_stream( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """When stream_options=include_usage is set, token usage comes from the final chunk's usage field (async).""" sentry_init( @@ -2153,8 +1617,7 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -2198,58 +1661,31 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + async for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - async for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - async for _ in response_stream: - pass - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_messages,expected_system_instructions,expected_output_tokens,expected_input_tokens", [ @@ -2336,7 +1772,6 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( ) def test_streaming_chat_completion( sentry_init, - capture_events, capture_items, get_messages, expected_system_instructions, @@ -2344,7 +1779,6 @@ def test_streaming_chat_completion( expected_input_tokens, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -2356,8 +1790,7 @@ def test_streaming_chat_completion( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -2407,135 +1840,71 @@ def test_streaming_chat_completion( include_event_type=False, ) ) - - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=get_messages(), - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - assert response_string == "hello world" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", + messages=get_messages(), + stream=True, + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert ( - "Message demonstrating the absence of truncation." - in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + response_string = "".join( + map(lambda x: x.choices[0].delta.content, response_stream) ) - assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert response_string == "hello world" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + == expected_system_instructions + ) - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - assert ( - span["attributes"]["gen_ai.usage.output_tokens"] - == expected_output_tokens - ) - assert ( - span["attributes"]["gen_ai.usage.input_tokens"] == expected_input_tokens - ) - assert ( - span["attributes"]["gen_ai.usage.total_tokens"] - == expected_output_tokens + expected_input_tokens - ) + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=get_messages(), - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - assert response_string == "hello world" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import assert ( - json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions + span["attributes"]["gen_ai.usage.output_tokens"] == expected_output_tokens + ) + assert span["attributes"]["gen_ai.usage.input_tokens"] == expected_input_tokens + assert ( + span["attributes"]["gen_ai.usage.total_tokens"] + == expected_output_tokens + expected_input_tokens ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert span["data"]["gen_ai.usage.output_tokens"] == expected_output_tokens - assert span["data"]["gen_ai.usage.input_tokens"] == expected_input_tokens - assert ( - span["data"]["gen_ai.usage.total_tokens"] - == expected_output_tokens + expected_input_tokens - ) - - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -2547,14 +1916,12 @@ def test_streaming_chat_completion( ) async def test_streaming_chat_completion_async_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -2566,8 +1933,7 @@ async def test_streaming_chat_completion_async_no_prompts( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -2619,122 +1985,63 @@ async def test_streaming_chat_completion_async_no_prompts( ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + stream=True, + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello world" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 - - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content - assert response_string == "hello world" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert response_string == "hello world" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 7 - assert span["data"]["gen_ai.usage.total_tokens"] == 9 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "get_messages,expected_system_instructions,expected_output_tokens,expected_input_tokens", @@ -2822,7 +2129,6 @@ async def test_streaming_chat_completion_async_no_prompts( ) async def test_streaming_chat_completion_async( sentry_init, - capture_events, capture_items, get_messages, expected_system_instructions, @@ -2831,7 +2137,6 @@ async def test_streaming_chat_completion_async( get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -2843,8 +2148,7 @@ async def test_streaming_chat_completion_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -2897,291 +2201,162 @@ async def test_streaming_chat_completion_async( ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", + messages=get_messages(), + stream=True, + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello world" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content + + assert response_string == "hello world" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + == expected_system_instructions + ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions - ) + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import assert ( - "Message demonstrating the absence of truncation." - in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + span["attributes"]["gen_ai.usage.output_tokens"] == expected_output_tokens ) - assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert ( - span["attributes"]["gen_ai.usage.output_tokens"] - == expected_output_tokens - ) - assert ( - span["attributes"]["gen_ai.usage.input_tokens"] == expected_input_tokens - ) - assert ( - span["attributes"]["gen_ai.usage.total_tokens"] - == expected_output_tokens + expected_input_tokens - ) - - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello world" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - + assert span["attributes"]["gen_ai.usage.input_tokens"] == expected_input_tokens assert ( - json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions + span["attributes"]["gen_ai.usage.total_tokens"] + == expected_output_tokens + expected_input_tokens ) - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert span["data"]["gen_ai.usage.output_tokens"] == expected_output_tokens - assert span["data"]["gen_ai.usage.input_tokens"] == expected_input_tokens - assert ( - span["data"]["gen_ai.usage.total_tokens"] - == expected_output_tokens + expected_input_tokens - ) - - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly -@pytest.mark.parametrize("span_streaming", [True, False]) def test_bad_chat_completion( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") - ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) - - (event,) = (item.payload for item in items if item.type == "event") - sentry_sdk.flush() - (span,) = (item.payload for item in items if item.type == "span") - assert event["level"] == "error" - assert span["status"] == "error" - else: - events = capture_events() - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + side_effect=OpenAIError("API rate limit reached") + ) + with pytest.raises(OpenAIError): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items if item.type == "event") + sentry_sdk.flush() + (span,) = (item.payload for item in items if item.type == "span") + assert event["level"] == "error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_status_error( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with start_transaction(name="test"): - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") + with start_transaction(name="test"): + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + side_effect=OpenAIError("API rate limit reached") + ) + with pytest.raises(OpenAIError): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) - - (error,) = (item.payload for item in items if item.type == "event") - assert error["level"] == "error" - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["status"] == "error" - else: - events = capture_events() - with start_transaction(name="test"): - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") - ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) + (error,) = (item.payload for item in items if item.type == "event") + assert error["level"] == "error" - (error, transaction) = events - assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_bad_chat_completion_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") client.chat.completions._post = AsyncMock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with pytest.raises(OpenAIError): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (event,) = (item.payload for item in items if item.type == "event") - sentry_sdk.flush() - (span,) = (item.payload for item in items if item.type == "span") - assert event["level"] == "error" - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(OpenAIError): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) + with pytest.raises(OpenAIError): + await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items if item.type == "event") + sentry_sdk.flush() + (span,) = (item.payload for item in items if item.type == "span") + assert event["level"] == "error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -3192,19 +2367,16 @@ async def test_bad_chat_completion_async( ) def test_embeddings_create_no_pii( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -3220,54 +2392,27 @@ def test_embeddings_create_no_pii( ) client.embeddings._post = mock.Mock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] - == "text-embedding-3-large" + with start_transaction(name="openai tx"): + response = client.embeddings.create( + input="hello", model="text-embedding-3-large" ) - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 + assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["data"] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["attributes"] - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_input,expected_embeddings_input", [ @@ -3339,19 +2484,16 @@ def test_embeddings_create_no_pii( ) def test_embeddings_create( sentry_init, - capture_events, capture_items, get_input, expected_embeddings_input, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -3367,90 +2509,44 @@ def test_embeddings_create( ) client.embeddings._post = mock.Mock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] - == "text-embedding-3-large" - ) - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) - == expected_embeddings_input - ) - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - - assert ( - json.loads(span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) - == expected_embeddings_input + with start_transaction(name="openai tx"): + response = client.embeddings.create( + input=get_input(), model="text-embedding-3-large" ) - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 - + assert len(response.data[0].embedding) == 3 -def _collect_embeddings_span_data( - capture_events, capture_items, span_streaming, create -): - if span_streaming: - items = capture_items("span") + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - with start_transaction(name="openai tx"): - response = create() + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) + == expected_embeddings_input + ) - assert len(response.data[0].embedding) == 3 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - return span["attributes"] - events = capture_events() +def _collect_embeddings_span_data(capture_events, capture_items, create): + items = capture_items("span") with start_transaction(name="openai tx"): response = create() assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - return span["data"] + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + return span["attributes"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,expect_input", [ @@ -3493,15 +2589,13 @@ def test_embeddings_create_data_collection( data_collection, send_default_pii, expect_input, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } sentry_init_kwargs = dict(init_kwargs) @@ -3527,7 +2621,6 @@ def test_embeddings_create_data_collection( span_data = _collect_embeddings_span_data( capture_events, capture_items, - span_streaming, lambda: client.embeddings.create(input="hello", model="text-embedding-3-large"), ) @@ -3544,7 +2637,6 @@ def test_embeddings_create_data_collection( assert span_data["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_input", [ @@ -3561,16 +2653,14 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( capture_events, capture_items, get_input, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", + trace_lifecycle="stream", _experiments={"data_collection": {"gen_ai": {"inputs": False}}}, - stream_gen_ai_spans=False, ) client = OpenAI(api_key="z") @@ -3590,7 +2680,6 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( span_data = _collect_embeddings_span_data( capture_events, capture_items, - span_streaming, lambda: client.embeddings.create( input=get_input(), model="text-embedding-3-large" ), @@ -3601,7 +2690,6 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -3613,19 +2701,16 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( ) async def test_embeddings_create_async_no_pii( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -3641,54 +2726,27 @@ async def test_embeddings_create_async_no_pii( ) client.embeddings._post = AsyncMock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] - == "text-embedding-3-large" + with start_transaction(name="openai tx"): + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" ) - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 + assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["data"] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["attributes"] - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "get_input,expected_embeddings_input", @@ -3761,19 +2819,16 @@ async def test_embeddings_create_async_no_pii( ) async def test_embeddings_create_async( sentry_init, - capture_events, capture_items, get_input, expected_embeddings_input, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -3789,60 +2844,30 @@ async def test_embeddings_create_async( ) client.embeddings._post = AsyncMock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] - == "text-embedding-3-large" - ) - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) - == expected_embeddings_input + with start_transaction(name="openai tx"): + response = await client.embeddings.create( + input=get_input(), model="text-embedding-3-large" ) - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 + assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - assert ( - json.loads(span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) - == expected_embeddings_input - ) + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) + == expected_embeddings_input + ) - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,expect_input", @@ -3875,20 +2900,17 @@ async def test_embeddings_create_async( ) async def test_embeddings_create_async_data_collection( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, expect_input, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } sentry_init_kwargs = dict(init_kwargs) @@ -3910,36 +2932,19 @@ async def test_embeddings_create_async_data_collection( ) client.embeddings._post = AsyncMock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + with start_transaction(name="openai tx"): + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) - assert len(response.data[0].embedding) == 3 + assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - span_data = span["data"] + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_SYSTEM] == "openai" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "embeddings" @@ -3954,26 +2959,22 @@ async def test_embeddings_create_async_data_collection( assert span_data["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [(True, True), (True, False), (False, True), (False, False)], ) def test_embeddings_create_raises_error( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -3981,30 +2982,18 @@ def test_embeddings_create_raises_error( client.embeddings._post = mock.Mock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with pytest.raises(OpenAIError): - client.embeddings.create(input="hello", model="text-embedding-3-large") - - (event,) = (item.payload for item in items if item.type == "event") - sentry_sdk.flush() - (span,) = (item.payload for item in items if item.type == "span") - assert event["level"] == "error" - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(OpenAIError): - client.embeddings.create(input="hello", model="text-embedding-3-large") + with pytest.raises(OpenAIError): + client.embeddings.create(input="hello", model="text-embedding-3-large") - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items if item.type == "event") + sentry_sdk.flush() + (span,) = (item.payload for item in items if item.type == "span") + assert event["level"] == "error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -4012,19 +3001,16 @@ def test_embeddings_create_raises_error( ) async def test_embeddings_create_raises_error_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -4032,46 +3018,27 @@ async def test_embeddings_create_raises_error_async( client.embeddings._post = AsyncMock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with pytest.raises(OpenAIError): - await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - (event,) = (item.payload for item in items if item.type == "event") - sentry_sdk.flush() - (span,) = (item.payload for item in items if item.type == "span") - assert event["level"] == "error" - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(OpenAIError): - await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + with pytest.raises(OpenAIError): + await client.embeddings.create(input="hello", model="text-embedding-3-large") - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items if item.type == "event") + sentry_sdk.flush() + (span,) = (item.payload for item in items if item.type == "span") + assert event["level"] == "error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_origin_nonstreaming_chat( sentry_init, - capture_events, capture_items, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -4088,48 +3055,30 @@ def test_span_origin_nonstreaming_chat( ), ) ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (event,) = events + with sentry_sdk.traces.start_span(name="openai tx"): + client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_origin_nonstreaming_chat_async( sentry_init, - capture_events, capture_items, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -4146,46 +3095,28 @@ async def test_span_origin_nonstreaming_chat_async( ), ) ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (event,) = events + with sentry_sdk.traces.start_span(name="openai tx"): + await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_origin_streaming_chat( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -4225,54 +3156,33 @@ def test_span_origin_streaming_chat( object="chat.completion.chunk", ), ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - client.chat.completions._post = mock.Mock(return_value=returned_stream) - with sentry_sdk.traces.start_span(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - "".join(map(lambda x: x.choices[0].delta.content, response_stream)) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - client.chat.completions._post = mock.Mock(return_value=returned_stream) - with start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - "".join(map(lambda x: x.choices[0].delta.content, response_stream)) + client.chat.completions._post = mock.Mock(return_value=returned_stream) + with sentry_sdk.traces.start_span(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - (event,) = events + "".join(map(lambda x: x.choices[0].delta.content, response_stream)) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_origin_streaming_chat_async( sentry_init, - capture_events, capture_items, async_iterator, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -4318,54 +3228,32 @@ async def test_span_origin_streaming_chat_async( ) client.chat.completions._post = AsyncMock(return_value=returned_stream) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - async for _ in response_stream: - pass - - # "".join(map(lambda x: x.choices[0].delta.content, response_stream)) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - async for _ in response_stream: - pass - - # "".join(map(lambda x: x.choices[0].delta.content, response_stream)) + with sentry_sdk.traces.start_span(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) + async for _ in response_stream: + pass - (event,) = events + # "".join(map(lambda x: x.choices[0].delta.content, response_stream)) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_origin_embeddings( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -4381,43 +3269,27 @@ def test_span_origin_embeddings( ) client.embeddings._post = mock.Mock(return_value=returned_embedding) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.embeddings.create(input="hello", model="text-embedding-3-large") - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.embeddings.create(input="hello", model="text-embedding-3-large") + with sentry_sdk.traces.start_span(name="openai tx"): + client.embeddings.create(input="hello", model="text-embedding-3-large") - (event,) = events + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_origin_embeddings_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -4433,31 +3305,15 @@ async def test_span_origin_embeddings_async( ) client.embeddings._post = AsyncMock(return_value=returned_embedding) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") + with sentry_sdk.traces.start_span(name="openai tx"): + await client.embeddings.create(input="hello", model="text-embedding-3-large") - with sentry_sdk.traces.start_span(name="openai tx"): - await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - (event,) = events - - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" def test_completions_token_usage_from_response(): @@ -4822,249 +3678,134 @@ def count_tokens(msg): ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_ai_client_span_responses_api_no_pii( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="openai tx"): + client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") + sentry_sdk.flush() + spans = [item.payload for item in items] + + assert len(spans) == 2 + expected_attributes = { + "gen_ai.operation.name": "responses", + "gen_ai.request.max_tokens": 100, + "gen_ai.request.temperature": 0.7, + "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.model": "response-model-id", + "gen_ai.response.streaming": False, + "gen_ai.system": "openai", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "sentry.op": "gen_ai.responses", + "sentry.origin": "auto.ai.openai", + "sentry.segment.name": "openai tx", + } - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) + for attr, value in expected_attributes.items(): + assert spans[0]["attributes"][attr] == value + + assert "gen_ai.system_instructions" not in spans[0]["attributes"] + assert "gen_ai.request.messages" not in spans[0]["attributes"] + assert "gen_ai.response.text" not in spans[0]["attributes"] - sentry_sdk.flush() - spans = [item.payload for item in items] - - assert len(spans) == 2 - expected_attributes = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.system": "openai", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "sentry.op": "gen_ai.responses", - "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", - } - - for attr, value in expected_attributes.items(): - assert spans[0]["attributes"][attr] == value - - assert "gen_ai.system_instructions" not in spans[0]["attributes"] - assert "gen_ai.request.messages" not in spans[0]["attributes"] - assert "gen_ai.response.text" not in spans[0]["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - assert spans[0]["origin"] == "auto.ai.openai" - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.system": "openai", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - } - - for key, value in expected_data.items(): - assert spans[0]["data"][key] == value - - assert "gen_ai.system_instructions" not in spans[0]["data"] - assert "gen_ai.request.messages" not in spans[0]["data"] - assert "gen_ai.response.text" not in spans[0]["data"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_ai_client_span_responses_tool_definitions( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) - - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="How do I check if a Python object is an instance of a class?", - tools=[ - FunctionToolParam( - type="function", - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="openai tx"): + client.responses.create( + model="gpt-4o", + input="How do I check if a Python object is an instance of a class?", + tools=[ + FunctionToolParam( + type="function", + name="name", + description="description", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, }, - strict=True, - ), - CustomToolParam( - type="custom", name="name", description="description" - ), - WebSearchToolParam(type="web_search"), - ], - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - assert json.loads(spans[0]["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ - { - "type": "function", - "name": "name", - "description": "description", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, + "required": ["city", "state"], + "additionalProperties": False, }, - "required": ["city", "state"], - "additionalProperties": False, - }, - }, - { - "type": "custom", - "name": "name", - "description": "description", - }, - { - "type": "web_search", - }, - ] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="How do I check if a Python object is an instance of a class?", - tools=[ - FunctionToolParam( - type="function", - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, - }, - strict=True, - ), - CustomToolParam( - type="custom", name="name", description="description" - ), - WebSearchToolParam(type="web_search"), - ], - ) - - (transaction,) = events - spans = transaction["spans"] + strict=True, + ), + CustomToolParam(type="custom", name="name", description="description"), + WebSearchToolParam(type="web_search"), + ], + ) - assert json.loads(spans[0]["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ - { - "type": "function", - "name": "name", - "description": "description", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, + sentry_sdk.flush() + spans = [item.payload for item in items] + assert json.loads(spans[0]["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + { + "type": "function", + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, }, + "required": ["city", "state"], + "additionalProperties": False, }, - { - "type": "custom", - "name": "name", - "description": "description", - }, - { - "type": "web_search", - }, - ] + }, + { + "type": "custom", + "name": "name", + "description": "description", + }, + { + "type": "web_search", + }, + ] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "instructions,input,expected_system_instructions,expected_request_messages", [ @@ -5211,125 +3952,71 @@ def test_ai_client_span_responses_tool_definitions( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_ai_client_span_responses_api( sentry_init, - capture_events, capture_items, instructions, input, expected_system_instructions, expected_request_messages, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="openai tx"): + client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - - assert len(spans) == 2 - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.system": "openai", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.request.messages": safe_serialize(expected_request_messages), - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.text": "the model response", - "sentry.op": "gen_ai.responses", - "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) - - for attr, value in expected_data.items(): - assert spans[0]["attributes"][attr] == value - - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + + assert len(spans) == 2 + + expected_data = { + "gen_ai.operation.name": "responses", + "gen_ai.request.max_tokens": 100, + "gen_ai.request.temperature": 0.7, + "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", + "gen_ai.system": "openai", + "gen_ai.response.model": "response-model-id", + "gen_ai.response.streaming": False, + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.request.messages": safe_serialize(expected_request_messages), + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.text": "the model response", + "sentry.op": "gen_ai.responses", + "sentry.origin": "auto.ai.openai", + "sentry.segment.name": "openai tx", + } - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.system": "openai", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.request.messages": safe_serialize(expected_request_messages[-1:]), - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.text": "the model response", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) + if expected_system_instructions is not None: + expected_data["gen_ai.system_instructions"] = safe_serialize( + expected_system_instructions + ) - for attr, value in expected_data.items(): - assert spans[0]["data"][attr] == value + for attr, value in expected_data.items(): + assert spans[0]["attributes"][attr] == value -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,extra_kwargs,expected_present,expected_absent,include_prompts", [ @@ -5456,22 +4143,19 @@ def test_ai_client_span_responses_api( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_responses_api_data_collection( sentry_init, - capture_events, capture_items, data_collection, extra_kwargs, expected_present, expected_absent, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -5485,30 +4169,16 @@ def test_responses_api_data_collection( "reasoning": {"effort": "high"}, } create_kwargs.update(extra_kwargs) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create(**create_kwargs) - - sentry_sdk.flush() - spans = [item.payload for item in items] + with sentry_sdk.traces.start_span(name="openai tx"): + client.responses.create(**create_kwargs) - assert len(spans) == 2 - span_data = spans[0]["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create(**create_kwargs) + sentry_sdk.flush() + spans = [item.payload for item in items] - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - span_data = spans[0]["data"] + assert len(spans) == 2 + span_data = spans[0]["attributes"] # Non-input data is always collected, regardless of data collection config assert span_data["gen_ai.operation.name"] == "responses" @@ -5571,29 +4241,17 @@ def _make_responses_api_response(output): ) -def _collect_responses_span_data(capture_events, capture_items, span_streaming, create): - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - create() - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - return span["attributes"] - - events = capture_events() +def _collect_responses_span_data(capture_events, capture_items, create): + items = capture_items("span") with start_transaction(name="openai tx"): create() - (transaction,) = events - (span,) = transaction["spans"] - assert span["op"] == "gen_ai.responses" - return span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + return span["attributes"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,expect_output", [ @@ -5637,15 +4295,13 @@ def test_responses_api_data_collection_outputs( data_collection, send_default_pii, expect_output, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } if data_collection is not None: init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -5673,7 +4329,6 @@ def test_responses_api_data_collection_outputs( span_data = _collect_responses_span_data( capture_events, capture_items, - span_streaming, lambda: client.responses.create(model="gpt-4o", input="hello"), ) @@ -5687,7 +4342,6 @@ def test_responses_api_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_output,expect_text,expect_tool_calls", [ @@ -5738,15 +4392,13 @@ def test_responses_api_data_collection_outputs_shapes( get_output, expect_text, expect_tool_calls, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, _experiments={"data_collection": {"gen_ai": {"outputs": True}}}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -5757,7 +4409,6 @@ def test_responses_api_data_collection_outputs_shapes( span_data = _collect_responses_span_data( capture_events, capture_items, - span_streaming, lambda: client.responses.create(model="gpt-4o", input="hello"), ) @@ -5772,7 +4423,6 @@ def test_responses_api_data_collection_outputs_shapes( assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,expect_output", [ @@ -5796,13 +4446,11 @@ def test_responses_api_data_collection_outputs_shapes( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_streaming_responses_api_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, expect_output, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], @@ -5810,8 +4458,7 @@ def test_streaming_responses_api_data_collection_outputs( traces_sample_rate=1.0, send_default_pii=False, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -5820,51 +4467,27 @@ def test_streaming_responses_api_data_collection_outputs( EXAMPLE_RESPONSES_STREAM, ) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.responses.create( + model="some-model", + input="hello", + stream=True, + ) + response_string = "" + for item in response_stream: + if hasattr(item, "delta"): + response_string += item.delta - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - response_string = "" - for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta - - assert response_string == "hello world" - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - response_string = "" - for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta - - assert response_string == "hello world" - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + assert response_string == "hello world" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] if expect_output: assert "hello world" in span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -5872,7 +4495,6 @@ def test_streaming_responses_api_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "conversation, expected_id", [ @@ -5885,60 +4507,37 @@ def test_streaming_responses_api_data_collection_outputs( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_responses_api_conversation_id( sentry_init, - capture_events, capture_items, conversation, expected_id, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - conversation=conversation, - ) + with start_transaction(name="openai tx"): + client.responses.create( + model="gpt-4o", + input="hello", + conversation=conversation, + ) - sentry_sdk.flush() - (span,) = (item.payload for item in items) + sentry_sdk.flush() + (span,) = (item.payload for item in items) - if expected_id is None: - assert "gen_ai.conversation.id" not in span["attributes"] - else: - assert span["attributes"]["gen_ai.conversation.id"] == expected_id + if expected_id is None: + assert "gen_ai.conversation.id" not in span["attributes"] else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - conversation=conversation, - ) - - (transaction,) = events - (span,) = transaction["spans"] + assert span["attributes"]["gen_ai.conversation.id"] == expected_id - if expected_id is None: - assert "gen_ai.conversation.id" not in span["data"] - else: - assert span["data"]["gen_ai.conversation.id"] == expected_id - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "reasoning, expected_level", [ @@ -5951,134 +4550,80 @@ def test_responses_api_conversation_id( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_responses_api_reasoning_level( sentry_init, - capture_events, capture_items, reasoning, expected_level, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - reasoning=reasoning, - ) - - sentry_sdk.flush() - span = next(item.payload for item in items if item.type == "span") + with start_transaction(name="openai tx"): + client.responses.create( + model="gpt-4o", + input="hello", + reasoning=reasoning, + ) - if expected_level is None: - assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] - else: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] - == expected_level - ) + sentry_sdk.flush() + span = next(item.payload for item in items if item.type == "span") + if expected_level is None: + assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - reasoning=reasoning, - ) - - (transaction,) = events - span = transaction["spans"][0] - - if expected_level is None: - assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["data"] - else: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == expected_level - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] + == expected_level + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_error_in_responses_api( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with sentry_sdk.traces.start_span(name="openai tx"), pytest.raises(OpenAIError): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) - - # make sure the span where the error occurred is captured - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["attributes"]["sentry.op"] == "gen_ai.responses" - - (error_event,) = (item.payload for item in items if item.type == "event") - - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - - assert spans[1]["is_segment"] is True - assert error_event["contexts"]["trace"]["trace_id"] == spans[1]["trace_id"] - else: - events = capture_events() + with sentry_sdk.traces.start_span(name="openai tx"), pytest.raises(OpenAIError): + client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) - with start_transaction(name="openai tx"), pytest.raises(OpenAIError): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) + # make sure the span where the error occurred is captured + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.op"] == "gen_ai.responses" - (error_event, transaction_event) = events + (error_event,) = (item.payload for item in items if item.type == "event") - assert transaction_event["type"] == "transaction" - # make sure the span where the error occurred is captured - assert transaction_event["spans"][0]["op"] == "gen_ai.responses" + assert error_event["level"] == "error" + assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert ( - error_event["contexts"]["trace"]["trace_id"] - == transaction_event["contexts"]["trace"]["trace_id"] - ) + assert spans[1]["is_segment"] is True + assert error_event["contexts"]["trace"]["trace_id"] == spans[1]["trace_id"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") @pytest.mark.parametrize( @@ -6226,125 +4771,71 @@ def test_error_in_responses_api( ) async def test_ai_client_span_responses_async_api( sentry_init, - capture_events, capture_items, instructions, input, expected_system_instructions, expected_request_messages, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") client.responses._post = AsyncMock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="openai tx"): + await client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - - assert len(spans) == 2 - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.request.messages": safe_serialize(expected_request_messages), - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.system": "openai", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - "sentry.op": "gen_ai.responses", - "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) - - for attr, value in expected_data.items(): - assert spans[0]["attributes"][attr] == value - - else: - events = capture_events() - - with start_transaction(name="openai tx"): - await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + + assert len(spans) == 2 + + expected_data = { + "gen_ai.operation.name": "responses", + "gen_ai.request.max_tokens": 100, + "gen_ai.request.temperature": 0.7, + "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", + "gen_ai.request.messages": safe_serialize(expected_request_messages), + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.model": "response-model-id", + "gen_ai.response.streaming": False, + "gen_ai.system": "openai", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.response.text": "the model response", + "sentry.op": "gen_ai.responses", + "sentry.origin": "auto.ai.openai", + "sentry.segment.name": "openai tx", + } - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.request.messages": safe_serialize(expected_request_messages[-1:]), - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.system": "openai", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) + if expected_system_instructions is not None: + expected_data["gen_ai.system_instructions"] = safe_serialize( + expected_system_instructions + ) - for attr, value in expected_data.items(): - assert spans[0]["data"][attr] == value + for attr, value in expected_data.items(): + assert spans[0]["attributes"][attr] == value -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "instructions,input,expected_system_instructions,expected_request_messages", @@ -6492,7 +4983,6 @@ async def test_ai_client_span_responses_async_api( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_ai_client_span_streaming_responses_async_api( sentry_init, - capture_events, capture_items, instructions, input, @@ -6501,215 +4991,122 @@ async def test_ai_client_span_streaming_responses_async_api( get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") returned_stream = get_model_response( async_iterator(server_side_event_chunks(EXAMPLE_RESPONSES_STREAM)) ) - - if span_streaming: - items = capture_items("span") - - ctx = ( - sentry_sdk.traces.start_span(name="openai tx") - if span_streaming - else start_transaction(name="openai tx") + items = capture_items("span") + + ctx = sentry_sdk.traces.start_span(name="openai tx") + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ), ctx: + result = await client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + stream=True, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, ) - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), ctx: - result = await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - async for _ in result: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items] - spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == OP.GEN_AI_RESPONSES - ] - - assert len(spans) == 1 - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.messages": safe_serialize(expected_request_messages), - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": True, - "gen_ai.system": "openai", - "gen_ai.response.time_to_first_token": mock.ANY, - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.text": "hello world", - "sentry.environment": "production", - "sentry.op": "gen_ai.responses", - "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) - - for attr, value in expected_data.items(): - assert spans[0]["attributes"][attr] == value - - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - result = await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - async for _ in result: - pass + async for _ in result: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items] + spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == OP.GEN_AI_RESPONSES + ] - (transaction,) = events - spans = [ - span for span in transaction["spans"] if span["op"] == OP.GEN_AI_RESPONSES - ] + assert len(spans) == 1 + + expected_data = { + "gen_ai.operation.name": "responses", + "gen_ai.request.max_tokens": 100, + "gen_ai.request.messages": safe_serialize(expected_request_messages), + "gen_ai.request.temperature": 0.7, + "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", + "gen_ai.response.model": "response-model-id", + "gen_ai.response.streaming": True, + "gen_ai.system": "openai", + "gen_ai.response.time_to_first_token": mock.ANY, + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.text": "hello world", + "sentry.environment": "production", + "sentry.op": "gen_ai.responses", + "sentry.origin": "auto.ai.openai", + "sentry.segment.name": "openai tx", + } - assert len(spans) == 1 - assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.messages": safe_serialize(expected_request_messages[-1:]), - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": True, - "gen_ai.system": "openai", - "gen_ai.response.time_to_first_token": mock.ANY, - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.text": "hello world", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) + if expected_system_instructions is not None: + expected_data["gen_ai.system_instructions"] = safe_serialize( + expected_system_instructions + ) - for attr, value in expected_data.items(): - assert spans[0]["data"][attr] == value + for attr, value in expected_data.items(): + assert spans[0]["attributes"][attr] == value -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_error_in_responses_async_api( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") client.responses._post = AsyncMock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with sentry_sdk.traces.start_span(name="openai tx"), pytest.raises(OpenAIError): - await client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) - - # make sure the span where the error occurred is captured - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["attributes"]["sentry.op"] == "gen_ai.responses" - - (error_event,) = (item.payload for item in items if item.type == "event") - - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - - assert spans[1]["is_segment"] is True - assert error_event["contexts"]["trace"]["trace_id"] == spans[1]["trace_id"] - else: - events = capture_events() + with sentry_sdk.traces.start_span(name="openai tx"), pytest.raises(OpenAIError): + await client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) - with start_transaction(name="openai tx"), pytest.raises(OpenAIError): - await client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) + # make sure the span where the error occurred is captured + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.op"] == "gen_ai.responses" - (error_event, transaction_event) = events + (error_event,) = (item.payload for item in items if item.type == "event") - assert transaction_event["type"] == "transaction" - # make sure the span where the error occurred is captured - assert transaction_event["spans"][0]["op"] == "gen_ai.responses" + assert error_event["level"] == "error" + assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert ( - error_event["contexts"]["trace"]["trace_id"] - == transaction_event["contexts"]["trace"]["trace_id"] - ) + assert spans[1]["is_segment"] is True + assert error_event["contexts"]["trace"]["trace_id"] == spans[1]["trace_id"] if SKIP_RESPONSES_TESTS: @@ -6786,7 +5183,6 @@ async def test_error_in_responses_async_api( ] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [(True, True), (True, False), (False, True), (False, False)], @@ -6794,13 +5190,11 @@ async def test_error_in_responses_async_api( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_streaming_responses_api( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -6811,8 +5205,7 @@ def test_streaming_responses_api( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -6821,102 +5214,53 @@ def test_streaming_responses_api( EXAMPLE_RESPONSES_STREAM, ) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.responses.create( + model="some-model", + input="hello", + stream=True, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - response_string = "" - for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta - - assert response_string == "hello world" - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.responses" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" - - if send_default_pii and include_prompts: - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - response_string = "" - for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta + response_string = "" + for item in response_stream: + if hasattr(item, "delta"): + response_string += item.delta - assert response_string == "hello world" + assert response_string == "hello world" - (transaction,) = events - (span,) = transaction["spans"] - assert span["op"] == "gen_ai.responses" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.responses" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" - if send_default_pii and include_prompts: - assert span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -6925,14 +5269,12 @@ def test_streaming_responses_api( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_streaming_responses_api_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -6943,110 +5285,60 @@ async def test_streaming_responses_api_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") returned_stream = get_model_response( async_iterator(server_side_event_chunks(EXAMPLE_RESPONSES_STREAM)) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = await client.responses.create( + model="some-model", + input="hello", + stream=True, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.responses.create( - model="some-model", - input="hello", - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - response_string = "" - async for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta - - assert response_string == "hello world" - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.responses" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" - - if send_default_pii and include_prompts: - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.responses.create( - model="some-model", - input="hello", - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - response_string = "" - async for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta + response_string = "" + async for item in response_stream: + if hasattr(item, "delta"): + response_string += item.delta - assert response_string == "hello world" + assert response_string == "hello world" - (transaction,) = events - (span,) = transaction["spans"] - assert span["op"] == "gen_ai.responses" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.responses" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" - if send_default_pii and include_prompts: - assert span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) # Feature added in https://github.com/openai/openai-python/pull/1952 @pytest.mark.skipif( OPENAI_VERSION is None or OPENAI_VERSION < (1, 58, 0), @@ -7063,19 +5355,16 @@ async def test_streaming_responses_api_async( ) def test_chat_completion_reasoning_level( sentry_init, - capture_events, capture_items, reasoning_effort, expected_level, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -7092,50 +5381,28 @@ def test_chat_completion_reasoning_level( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - reasoning_effort=reasoning_effort, - ) + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], + reasoning_effort=reasoning_effort, + ) - sentry_sdk.flush() - span = next(item.payload for item in items if item.type == "span") + sentry_sdk.flush() + span = next(item.payload for item in items if item.type == "span") - if expected_level is None: - assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] - else: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] - == expected_level - ) + if expected_level is None: + assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - reasoning_effort=reasoning_effort, - ) - - (event,) = events - span = event["spans"][0] - - if expected_level is None: - assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["data"] - else: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == expected_level - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] + == expected_level + ) # Test messages with mixed roles including "ai" that should be mapped to "assistant" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "test_message,expected_role", [ @@ -7152,12 +5419,10 @@ def test_chat_completion_reasoning_level( ) def test_openai_message_role_mapping( sentry_init, - capture_events, capture_items, test_message, expected_role, nonstreaming_chat_completions_model_response, - span_streaming, ): """Test that OpenAI integration properly maps message roles like 'ai' to 'assistant'""" @@ -7166,8 +5431,7 @@ def test_openai_message_role_mapping( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -7186,112 +5450,29 @@ def test_openai_message_role_mapping( ) test_messages = [test_message] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create(model="test-model", messages=test_messages) - - # Verify that the span was created correctly - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create(model="test-model", messages=test_messages) + with start_transaction(name="openai tx"): + client.chat.completions.create(model="test-model", messages=test_messages) - # Verify that the span was created correctly - (event,) = events - span = event["spans"][0] - assert span["op"] == "gen_ai.chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] + # Verify that the span was created correctly + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert len(stored_messages) == 1 assert stored_messages[0]["role"] == expected_role -def test_openai_message_truncation( - sentry_init, - capture_events, - nonstreaming_chat_completions_model_response, -): - """Test that large messages are truncated properly in OpenAI integration.""" - sentry_init( - integrations=[OpenAIIntegration(include_prompts=True)], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - return_value=nonstreaming_chat_completions_model_response( - response_id="chat-id", - response_model="gpt-3.5-turbo", - message_content="the model response", - created=10000000, - usage=CompletionUsage( - prompt_tokens=20, - completion_tokens=10, - total_tokens=30, - ), - ) - ) - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - large_messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": large_content}, - {"role": "assistant", "content": large_content}, - {"role": "user", "content": large_content}, - ] - - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=large_messages, - ) - - (event,) = events - span = event["spans"][0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - - messages_data = span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) <= len(large_messages) - - meta_path = event["_meta"] - span_meta = meta_path["spans"]["0"]["data"] - messages_meta = span_meta[SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "len" in messages_meta.get("", {}) - - # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_chat_completion_ttft( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """ Test that streaming chat completions capture time-to-first-token (TTFT). @@ -7300,8 +5481,7 @@ def test_streaming_chat_completion_ttft( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -7338,71 +5518,42 @@ def test_streaming_chat_completion_ttft( include_event_type=False, ), ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + ) + # Consume the stream + for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "Say hello"}], - stream=True, - ) - # Consume the stream - for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] - ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "Say hello"}], - stream=True, - ) - # Consume the stream - for _ in response_stream: - pass - - (tx,) = events - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["data"] - ttft = span["data"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] + # Verify TTFT is captured + assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] + ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] assert isinstance(ttft, float) assert ttft > 0 # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_streaming_chat_completion_ttft_async( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """ Test that async streaming chat completions capture time-to-first-token (TTFT). @@ -7411,8 +5562,7 @@ async def test_streaming_chat_completion_ttft_async( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -7451,70 +5601,41 @@ async def test_streaming_chat_completion_ttft_async( ), ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + ) + # Consume the stream + async for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "Say hello"}], - stream=True, - ) - # Consume the stream - async for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] - ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "Say hello"}], - stream=True, - ) - # Consume the stream - async for _ in response_stream: - pass - - (tx,) = events - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["data"] - ttft = span["data"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] + # Verify TTFT is captured + assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] + ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] assert isinstance(ttft, float) assert ttft > 0 # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_streaming_responses_api_ttft( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """ Test that streaming responses API captures time-to-first-token (TTFT). @@ -7523,80 +5644,50 @@ def test_streaming_responses_api_ttft( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") returned_stream = get_model_response( server_side_event_chunks(EXAMPLE_RESPONSES_STREAM) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = client.responses.create( + model="some-model", + input="hello", + stream=True, + ) + # Consume the stream + for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - # Consume the stream - for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.responses" - - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] - ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - # Consume the stream - for _ in response_stream: - pass - - (tx,) = events - span = tx["spans"][0] - assert span["op"] == "gen_ai.responses" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.responses" - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["data"] - ttft = span["data"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] + # Verify TTFT is captured + assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] + ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] assert isinstance(ttft, float) assert ttft > 0 # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_streaming_responses_api_ttft_async( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """ Test that async streaming responses API captures time-to-first-token (TTFT). @@ -7605,63 +5696,36 @@ async def test_streaming_responses_api_ttft_async( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") returned_stream = get_model_response( async_iterator(server_side_event_chunks(EXAMPLE_RESPONSES_STREAM)) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ), start_transaction(name="openai tx"): + response_stream = await client.responses.create( + model="some-model", + input="hello", + stream=True, + ) + # Consume the stream + async for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - # Consume the stream - async for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.responses" - - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] - ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - # Consume the stream - async for _ in response_stream: - pass - - (tx,) = events - span = tx["spans"][0] - assert span["op"] == "gen_ai.responses" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.responses" - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["data"] - ttft = span["data"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] + # Verify TTFT is captured + assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] + ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] assert isinstance(ttft, float) assert ttft > 0 From 6e9a7a000d62e3f642cf2254cdda8bdf144f0786 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 3 Sep 2026 13:59:54 +0200 Subject: [PATCH 2/5] remove truncation --- sentry_sdk/integrations/openai.py | 85 +++++++++++-------------------- 1 file changed, 30 insertions(+), 55 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index b90a29c3fa..3491b02ee0 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -38,16 +38,11 @@ from sentry_sdk.ai.utils import ( normalize_message_roles, set_data_normalized, - truncate_and_annotate_embedding_inputs, - truncate_and_annotate_messages, ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import StreamedSpan -from sentry_sdk.tracing_utils import ( - has_span_streaming_enabled, -) from sentry_sdk.utils import ( capture_internal_exceptions, event_from_exception, @@ -447,16 +442,12 @@ def _set_responses_api_input_data( # Input was provided as a single string if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore - client = sentry_sdk.get_client() - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) return @@ -466,16 +457,12 @@ def _set_responses_api_input_data( ] if len(non_system_messages) > 0: normalized_messages = normalize_message_roles(non_system_messages) # type: ignore - client = sentry_sdk.get_client() - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) @@ -550,15 +537,12 @@ def _set_completions_api_input_data( if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) return @@ -584,15 +568,12 @@ def _set_completions_api_input_data( if len(non_system_messages) > 0: normalized_messages = normalize_message_roles(non_system_messages) # type: ignore client = sentry_sdk.get_client() - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) @@ -624,15 +605,12 @@ def _set_embeddings_input_data( if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_embedding_inputs(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False + span, + SPANDATA.GEN_AI_EMBEDDINGS_INPUT, + normalized_messages, + unpack=False, ) return @@ -646,15 +624,12 @@ def _set_embeddings_input_data( if len(messages_copy) > 0: normalized_messages = normalize_message_roles(messages_copy) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_embedding_inputs(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False + span, + SPANDATA.GEN_AI_EMBEDDINGS_INPUT, + normalized_messages, + unpack=False, ) From 0619edae5bbbf984531c37e738835c639b463722 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 3 Sep 2026 15:10:48 +0200 Subject: [PATCH 3/5] remove start_transaction --- tests/integrations/openai/test_openai.py | 326 +++++++++++------------ 1 file changed, 150 insertions(+), 176 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index e2700ea576..5c9cb06173 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -63,7 +63,6 @@ from unittest import mock # python 3.3 and above -from sentry_sdk import start_transaction from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.openai import ( OpenAIIntegration, @@ -196,40 +195,39 @@ def test_chat_completion_tool_definitions( ) items = capture_items("span") - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - tools=[ - ChatCompletionFunctionToolParam( - type="function", - function=FunctionDefinition( - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + tools=[ + ChatCompletionFunctionToolParam( + type="function", + function=FunctionDefinition( + name="name", + description="description", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, }, - strict=True, - ), + "required": ["city", "state"], + "additionalProperties": False, + }, + strict=True, ), - ChatCompletionCustomToolParam( - type="custom", - custom=Custom( - name="name", - description="description", - ), + ), + ChatCompletionCustomToolParam( + type="custom", + custom=Custom( + name="name", + description="description", ), - ], - ) + ), + ], + ) sentry_sdk.flush() span = next(item.payload for item in items) @@ -296,23 +294,22 @@ def test_nonstreaming_chat_completion_no_prompts( ) items = capture_items("span") - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content + response = ( + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) + .choices[0] + .message.content + ) assert response == "the model response" sentry_sdk.flush() @@ -446,20 +443,19 @@ def test_nonstreaming_chat_completion( ) items = capture_items("span") - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content + response = ( + client.chat.completions.create( + model="some-model", + messages=get_messages(), + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) + .choices[0] + .message.content + ) assert response == "the model response" sentry_sdk.flush() @@ -585,8 +581,7 @@ def test_completions_api_data_collection( } items = capture_items("span") - with start_transaction(name="openai tx"): - client.chat.completions.create(**create_kwargs) + client.chat.completions.create(**create_kwargs) sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -685,11 +680,10 @@ def test_completions_api_data_collection_outputs( ) items = capture_items("span") - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -781,11 +775,10 @@ async def test_completions_api_data_collection_outputs_async( ) items = capture_items("span") - with start_transaction(name="openai tx"): - await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -828,11 +821,10 @@ def test_completions_api_data_collection_outputs_empty_choices( ) items = capture_items("span") - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -903,10 +895,8 @@ def test_streaming_chat_completion_data_collection_outputs( items = capture_items("span") with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): + client.chat._client._client, "send", return_value=returned_stream + ): response_stream = client.chat.completions.create( model="some-model", messages=[{"role": "user", "content": "hello"}], @@ -995,7 +985,7 @@ async def test_streaming_chat_completion_data_collection_outputs_async( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = await client.chat.completions.create( model="some-model", messages=[{"role": "user", "content": "hello"}], @@ -1056,20 +1046,19 @@ async def test_nonstreaming_chat_completion_async_no_prompts( ) items = capture_items("span") - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content + response = await client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) + response = response.choices[0].message.content assert response == "the model response" sentry_sdk.flush() @@ -1204,17 +1193,16 @@ async def test_nonstreaming_chat_completion_async( ) items = capture_items("span") - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content + response = await client.chat.completions.create( + model="some-model", + messages=get_messages(), + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) + response = response.choices[0].message.content assert response == "the model response" sentry_sdk.flush() @@ -1339,7 +1327,7 @@ def test_streaming_chat_completion_no_prompts( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = client.chat.completions.create( model="some-model", messages=[ @@ -1451,7 +1439,7 @@ def test_streaming_chat_completion_with_usage_in_stream( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = client.chat.completions.create( model="some-model", messages=[{"role": "user", "content": "hello"}], @@ -1514,7 +1502,7 @@ def test_streaming_chat_completion_empty_content_preserves_token_usage( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = client.chat.completions.create( model="some-model", messages=[{"role": "user", "content": "hello"}], @@ -1581,7 +1569,7 @@ async def test_streaming_chat_completion_empty_content_preserves_token_usage_asy client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = await client.chat.completions.create( model="some-model", messages=[{"role": "user", "content": "hello"}], @@ -1667,7 +1655,7 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = await client.chat.completions.create( model="some-model", messages=[{"role": "user", "content": "hello"}], @@ -1846,7 +1834,7 @@ def test_streaming_chat_completion( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = client.chat.completions.create( model="some-model", messages=get_messages(), @@ -1991,7 +1979,7 @@ async def test_streaming_chat_completion_async_no_prompts( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = await client.chat.completions.create( model="some-model", messages=[ @@ -2207,7 +2195,7 @@ async def test_streaming_chat_completion_async( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = await client.chat.completions.create( model="some-model", messages=get_messages(), @@ -2308,16 +2296,15 @@ def test_span_status_error( ) items = capture_items("event", "span") - with start_transaction(name="test"): - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + side_effect=OpenAIError("API rate limit reached") + ) + with pytest.raises(OpenAIError): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) (error,) = (item.payload for item in items if item.type == "event") assert error["level"] == "error" @@ -2394,10 +2381,7 @@ def test_embeddings_create_no_pii( client.embeddings._post = mock.Mock(return_value=returned_embedding) items = capture_items("span") - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + response = client.embeddings.create(input="hello", model="text-embedding-3-large") assert len(response.data[0].embedding) == 3 @@ -2511,10 +2495,9 @@ def test_embeddings_create( client.embeddings._post = mock.Mock(return_value=returned_embedding) items = capture_items("span") - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) + response = client.embeddings.create( + input=get_input(), model="text-embedding-3-large" + ) assert len(response.data[0].embedding) == 3 @@ -2536,8 +2519,7 @@ def test_embeddings_create( def _collect_embeddings_span_data(capture_events, capture_items, create): items = capture_items("span") - with start_transaction(name="openai tx"): - response = create() + response = create() assert len(response.data[0].embedding) == 3 @@ -2728,10 +2710,9 @@ async def test_embeddings_create_async_no_pii( client.embeddings._post = AsyncMock(return_value=returned_embedding) items = capture_items("span") - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) assert len(response.data[0].embedding) == 3 @@ -2846,10 +2827,9 @@ async def test_embeddings_create_async( client.embeddings._post = AsyncMock(return_value=returned_embedding) items = capture_items("span") - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) + response = await client.embeddings.create( + input=get_input(), model="text-embedding-3-large" + ) assert len(response.data[0].embedding) == 3 @@ -2934,10 +2914,9 @@ async def test_embeddings_create_async_data_collection( client.embeddings._post = AsyncMock(return_value=returned_embedding) items = capture_items("span") - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) assert len(response.data[0].embedding) == 3 @@ -4244,8 +4223,7 @@ def _make_responses_api_response(output): def _collect_responses_span_data(capture_events, capture_items, create): items = capture_items("span") - with start_transaction(name="openai tx"): - create() + create() sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -4473,7 +4451,7 @@ def test_streaming_responses_api_data_collection_outputs( client.responses._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = client.responses.create( model="some-model", input="hello", @@ -4522,12 +4500,11 @@ def test_responses_api_conversation_id( client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) items = capture_items("span") - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - conversation=conversation, - ) + client.responses.create( + model="gpt-4o", + input="hello", + conversation=conversation, + ) sentry_sdk.flush() (span,) = (item.payload for item in items) @@ -4565,12 +4542,11 @@ def test_responses_api_reasoning_level( client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) items = capture_items("span") - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - reasoning=reasoning, - ) + client.responses.create( + model="gpt-4o", + input="hello", + reasoning=reasoning, + ) sentry_sdk.flush() span = next(item.payload for item in items if item.type == "span") @@ -5220,7 +5196,7 @@ def test_streaming_responses_api( client.responses._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = client.responses.create( model="some-model", input="hello", @@ -5298,7 +5274,7 @@ async def test_streaming_responses_api_async( client.responses._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = await client.responses.create( model="some-model", input="hello", @@ -5383,12 +5359,11 @@ def test_chat_completion_reasoning_level( ) items = capture_items("span") - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - reasoning_effort=reasoning_effort, - ) + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], + reasoning_effort=reasoning_effort, + ) sentry_sdk.flush() span = next(item.payload for item in items if item.type == "span") @@ -5452,8 +5427,7 @@ def test_openai_message_role_mapping( test_messages = [test_message] items = capture_items("span") - with start_transaction(name="openai tx"): - client.chat.completions.create(model="test-model", messages=test_messages) + client.chat.completions.create(model="test-model", messages=test_messages) # Verify that the span was created correctly sentry_sdk.flush() @@ -5524,7 +5498,7 @@ def test_streaming_chat_completion_ttft( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = client.chat.completions.create( model="some-model", messages=[{"role": "user", "content": "Say hello"}], @@ -5607,7 +5581,7 @@ async def test_streaming_chat_completion_ttft_async( client.chat._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = await client.chat.completions.create( model="some-model", messages=[{"role": "user", "content": "Say hello"}], @@ -5657,7 +5631,7 @@ def test_streaming_responses_api_ttft( client.responses._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = client.responses.create( model="some-model", input="hello", @@ -5709,7 +5683,7 @@ async def test_streaming_responses_api_ttft_async( client.responses._client._client, "send", return_value=returned_stream, - ), start_transaction(name="openai tx"): + ): response_stream = await client.responses.create( model="some-model", input="hello", From 236cb6f0a1cf34dfda3a9b13de1e9f81be1ea309 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 3 Sep 2026 15:49:21 +0200 Subject: [PATCH 4/5] remove all manual spans --- tests/integrations/openai/test_openai.py | 190 ++++++++++------------- 1 file changed, 83 insertions(+), 107 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 5c9cb06173..2894180dea 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -3036,14 +3036,12 @@ def test_span_origin_nonstreaming_chat( ) items = capture_items("transaction", "span") - with sentry_sdk.traces.start_span(name="openai tx"): - client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) + client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" @@ -3076,14 +3074,12 @@ async def test_span_origin_nonstreaming_chat_async( ) items = capture_items("transaction", "span") - with sentry_sdk.traces.start_span(name="openai tx"): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) + await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" @@ -3138,16 +3134,14 @@ def test_span_origin_streaming_chat( items = capture_items("transaction", "span") client.chat.completions._post = mock.Mock(return_value=returned_stream) - with sentry_sdk.traces.start_span(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) + response_stream = client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - "".join(map(lambda x: x.choices[0].delta.content, response_stream)) + "".join(map(lambda x: x.choices[0].delta.content, response_stream)) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" @@ -3209,18 +3203,16 @@ async def test_span_origin_streaming_chat_async( client.chat.completions._post = AsyncMock(return_value=returned_stream) items = capture_items("transaction", "span") - with sentry_sdk.traces.start_span(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - async for _ in response_stream: - pass + response_stream = await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) + async for _ in response_stream: + pass - # "".join(map(lambda x: x.choices[0].delta.content, response_stream)) + # "".join(map(lambda x: x.choices[0].delta.content, response_stream)) sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" @@ -3250,12 +3242,10 @@ def test_span_origin_embeddings( client.embeddings._post = mock.Mock(return_value=returned_embedding) items = capture_items("transaction", "span") - with sentry_sdk.traces.start_span(name="openai tx"): - client.embeddings.create(input="hello", model="text-embedding-3-large") + client.embeddings.create(input="hello", model="text-embedding-3-large") sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" @@ -3286,12 +3276,10 @@ async def test_span_origin_embeddings_async( client.embeddings._post = AsyncMock(return_value=returned_embedding) items = capture_items("transaction", "span") - with sentry_sdk.traces.start_span(name="openai tx"): - await client.embeddings.create(input="hello", model="text-embedding-3-large") + await client.embeddings.create(input="hello", model="text-embedding-3-large") sentry_sdk.flush() spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" @@ -3673,21 +3661,20 @@ def test_ai_client_span_responses_api_no_pii( client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) items = capture_items("span") - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) + client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 expected_attributes = { "gen_ai.operation.name": "responses", "gen_ai.request.max_tokens": 100, @@ -3705,7 +3692,6 @@ def test_ai_client_span_responses_api_no_pii( "gen_ai.usage.total_tokens": 30, "sentry.op": "gen_ai.responses", "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", } for attr, value in expected_attributes.items(): @@ -3732,30 +3718,29 @@ def test_ai_client_span_responses_tool_definitions( client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) items = capture_items("span") - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="How do I check if a Python object is an instance of a class?", - tools=[ - FunctionToolParam( - type="function", - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, + client.responses.create( + model="gpt-4o", + input="How do I check if a Python object is an instance of a class?", + tools=[ + FunctionToolParam( + type="function", + name="name", + description="description", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, }, - strict=True, - ), - CustomToolParam(type="custom", name="name", description="description"), - WebSearchToolParam(type="web_search"), - ], - ) + "required": ["city", "state"], + "additionalProperties": False, + }, + strict=True, + ), + CustomToolParam(type="custom", name="name", description="description"), + WebSearchToolParam(type="web_search"), + ], + ) sentry_sdk.flush() spans = [item.payload for item in items] @@ -3949,21 +3934,20 @@ def test_ai_client_span_responses_api( client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) items = capture_items("span") - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) + client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 expected_data = { "gen_ai.operation.name": "responses", @@ -3984,7 +3968,6 @@ def test_ai_client_span_responses_api( "gen_ai.response.text": "the model response", "sentry.op": "gen_ai.responses", "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", } if expected_system_instructions is not None: @@ -4150,13 +4133,12 @@ def test_responses_api_data_collection( create_kwargs.update(extra_kwargs) items = capture_items("span") - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create(**create_kwargs) + client.responses.create(**create_kwargs) sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 span_data = spans[0]["attributes"] # Non-input data is always collected, regardless of data collection config @@ -4579,12 +4561,11 @@ def test_error_in_responses_api( ) items = capture_items("event", "span") - with sentry_sdk.traces.start_span(name="openai tx"), pytest.raises(OpenAIError): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) + client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) # make sure the span where the error occurred is captured sentry_sdk.flush() @@ -4765,21 +4746,20 @@ async def test_ai_client_span_responses_async_api( client.responses._post = AsyncMock(return_value=EXAMPLE_RESPONSE) items = capture_items("span") - with sentry_sdk.traces.start_span(name="openai tx"): - await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) + await client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) sentry_sdk.flush() spans = [item.payload for item in items] - assert len(spans) == 2 + assert len(spans) == 1 expected_data = { "gen_ai.operation.name": "responses", @@ -4800,7 +4780,6 @@ async def test_ai_client_span_responses_async_api( "gen_ai.response.text": "the model response", "sentry.op": "gen_ai.responses", "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", } if expected_system_instructions is not None: @@ -4982,12 +4961,11 @@ async def test_ai_client_span_streaming_responses_async_api( ) items = capture_items("span") - ctx = sentry_sdk.traces.start_span(name="openai tx") with mock.patch.object( client.responses._client._client, "send", return_value=returned_stream, - ), ctx: + ): result = await client.responses.create( model="gpt-4o", instructions=instructions, @@ -5032,7 +5010,6 @@ async def test_ai_client_span_streaming_responses_async_api( "sentry.environment": "production", "sentry.op": "gen_ai.responses", "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", } if expected_system_instructions is not None: @@ -5064,12 +5041,11 @@ async def test_error_in_responses_async_api( ) items = capture_items("event", "span") - with sentry_sdk.traces.start_span(name="openai tx"), pytest.raises(OpenAIError): - await client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) + await client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) # make sure the span where the error occurred is captured sentry_sdk.flush() From e723122e8412db30e8f04bf174fdce8f4c680a95 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 4 Sep 2026 09:44:32 +0200 Subject: [PATCH 5/5] restore with pytest.raises(OpenAIError): --- tests/integrations/openai/test_openai.py | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 2894180dea..c7dfbebf0e 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -4561,11 +4561,12 @@ def test_error_in_responses_api( ) items = capture_items("event", "span") - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) + with pytest.raises(OpenAIError): + client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) # make sure the span where the error occurred is captured sentry_sdk.flush() @@ -4577,8 +4578,7 @@ def test_error_in_responses_api( assert error_event["level"] == "error" assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert spans[1]["is_segment"] is True - assert error_event["contexts"]["trace"]["trace_id"] == spans[1]["trace_id"] + assert error_event["contexts"]["trace"]["trace_id"] == spans[0]["trace_id"] @pytest.mark.asyncio @@ -5041,11 +5041,12 @@ async def test_error_in_responses_async_api( ) items = capture_items("event", "span") - await client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) + with pytest.raises(OpenAIError): + await client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) # make sure the span where the error occurred is captured sentry_sdk.flush() @@ -5057,8 +5058,7 @@ async def test_error_in_responses_async_api( assert error_event["level"] == "error" assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert spans[1]["is_segment"] is True - assert error_event["contexts"]["trace"]["trace_id"] == spans[1]["trace_id"] + assert error_event["contexts"]["trace"]["trace_id"] == spans[0]["trace_id"] if SKIP_RESPONSES_TESTS: