diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 5ed2e9172d..3491b02ee0 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -36,19 +36,13 @@ ) from sentry_sdk.ai.monitoring import record_token_usage from sentry_sdk.ai.utils import ( - get_start_span_function, normalize_message_roles, set_data_normalized, - truncate_and_annotate_embedding_inputs, - truncate_and_annotate_messages, ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import StreamedSpan -from sentry_sdk.tracing_utils import ( - has_span_streaming_enabled, -) from sentry_sdk.utils import ( capture_internal_exceptions, event_from_exception, @@ -80,7 +74,6 @@ from openai.types.responses.response_usage import ResponseUsage from sentry_sdk._types import TextPart - from sentry_sdk.tracing import Span try: try: @@ -181,7 +174,7 @@ def _has_attr_and_is_int( def _calculate_completions_token_usage( messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]", response: "Any", - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", streaming_message_responses: "Optional[List[str]]", streaming_message_total_token_usage: "Optional[CompletionUsage]", count_tokens: "Callable[..., Any]", @@ -263,7 +256,7 @@ def _calculate_completions_token_usage( def _calculate_responses_token_usage( input: "Any", response: "Any", - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", streaming_message_responses: "Optional[List[str]]", count_tokens: "Callable[..., Any]", ) -> None: @@ -339,31 +332,27 @@ def _calculate_responses_token_usage( def _set_responses_api_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") model = kwargs.get("model") if model is not None: - set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model) max_tokens = kwargs.get("max_output_tokens") if max_tokens is not None and _is_given(max_tokens): - set_on_span(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) temperature = kwargs.get("temperature") if temperature is not None and _is_given(temperature): - set_on_span(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) top_p = kwargs.get("top_p") if top_p is not None and _is_given(top_p): - set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) conversation = kwargs.get("conversation") if conversation is not None and _is_given(conversation): @@ -373,11 +362,11 @@ def _set_responses_api_input_data( elif isinstance(conversation, dict): conversation_id = conversation.get("id") if conversation_id is not None: - set_on_span(SPANDATA.GEN_AI_CONVERSATION_ID, conversation_id) + span.set_attribute(SPANDATA.GEN_AI_CONVERSATION_ID, conversation_id) reasoning = kwargs.get("reasoning") if isinstance(reasoning, dict) and "effort" in reasoning: - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL, reasoning["effort"], ) @@ -387,7 +376,7 @@ def _set_responses_api_input_data( if client_options["data_collection"]["gen_ai"]["inputs"]: tools = kwargs.get("tools") if tools is not None and _is_given(tools): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions_responses(tools)), ) @@ -398,7 +387,7 @@ def _set_responses_api_input_data( # line below tools = kwargs.get("tools") if tools is not None and _is_given(tools): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions_responses(tools)), ) @@ -418,7 +407,7 @@ def _set_responses_api_input_data( if messages is None: if has_explicit_instructions: - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps( [ @@ -445,7 +434,7 @@ def _set_responses_api_input_data( system_instructions ) if len(instructions_text_parts) > 0: - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps(instructions_text_parts), ) @@ -453,16 +442,12 @@ def _set_responses_api_input_data( # Input was provided as a single string if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore - client = sentry_sdk.get_client() - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) return @@ -472,64 +457,56 @@ def _set_responses_api_input_data( ] if len(non_system_messages) > 0: normalized_messages = normalize_message_roles(non_system_messages) # type: ignore - client = sentry_sdk.get_client() - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) def _set_completions_api_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") model = kwargs.get("model") if model is not None: - set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model) max_tokens = kwargs.get("max_tokens") if max_tokens is not None and _is_given(max_tokens): - set_on_span(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) presence_penalty = kwargs.get("presence_penalty") if presence_penalty is not None and _is_given(presence_penalty): - set_on_span(SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, presence_penalty) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, presence_penalty) frequency_penalty = kwargs.get("frequency_penalty") if frequency_penalty is not None and _is_given(frequency_penalty): - set_on_span(SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, frequency_penalty) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, frequency_penalty) temperature = kwargs.get("temperature") if temperature is not None and _is_given(temperature): - set_on_span(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature) top_p = kwargs.get("top_p") if top_p is not None and _is_given(top_p): - set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) reasoning_level = kwargs.get("reasoning_effort") if reasoning_level is not None and _is_given(reasoning_level): - set_on_span(SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL, reasoning_level) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL, reasoning_level) client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): if client.options["data_collection"]["gen_ai"]["inputs"]: tools = kwargs.get("tools") if tools is not None and _is_given(tools): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions_completions(tools)), ) @@ -540,7 +517,7 @@ def _set_completions_api_input_data( # line below tools = kwargs.get("tools") if tools is not None and _is_given(tools): - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions_completions(tools)), ) @@ -560,15 +537,12 @@ def _set_completions_api_input_data( if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) return @@ -581,7 +555,7 @@ def _set_completions_api_input_data( system_instructions = _get_system_instructions_completions(messages) if len(system_instructions) > 0: - set_on_span( + span.set_attribute( SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, json.dumps(_transform_system_instructions_completions(system_instructions)), ) @@ -594,33 +568,26 @@ def _set_completions_api_input_data( if len(non_system_messages) > 0: normalized_messages = normalize_message_roles(non_system_messages) # type: ignore client = sentry_sdk.get_client() - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_messages(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) def _set_embeddings_input_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") - set_on_span = ( - span.set_attribute if isinstance(span, StreamedSpan) else span.set_data - ) - model = kwargs.get("model") if model is not None: - set_on_span(SPANDATA.GEN_AI_REQUEST_MODEL, model) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model) messages: "Optional[Union[str, SequenceNotStr[str], Iterable[int], Iterable[Iterable[int]]]]" = kwargs.get( "input" @@ -638,15 +605,12 @@ def _set_embeddings_input_data( if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_embedding_inputs(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False + span, + SPANDATA.GEN_AI_EMBEDDINGS_INPUT, + normalized_messages, + unpack=False, ) return @@ -660,20 +624,17 @@ def _set_embeddings_input_data( if len(messages_copy) > 0: normalized_messages = normalize_message_roles(messages_copy) # type: ignore - scope = sentry_sdk.get_current_scope() - messages_data = ( - truncate_and_annotate_embedding_inputs(normalized_messages, span, scope) - if not has_span_streaming_enabled(client.options) - else normalized_messages - ) - if messages_data is not None: + if normalized_messages is not None: set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False + span, + SPANDATA.GEN_AI_EMBEDDINGS_INPUT, + normalized_messages, + unpack=False, ) def _set_common_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", response: "Union[ChatCompletion, Stream[ChatCompletionChunk], AsyncStream[ChatCompletionChunk], Response, Stream[ResponseStreamEvent], AsyncStream[ResponseStreamEvent], CreateEmbeddingResponse]", input: "Any", integration: "OpenAIIntegration", @@ -836,27 +797,15 @@ def _new_sync_chat_completion( # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/completions.py#L585 is_streaming_response = kwargs.get("stream", False) or False - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"chat {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_CHAT, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, - }, - ) - - else: - span = get_start_span_function()( - op=consts.OP.GEN_AI_CHAT, - name=f"chat {model}", - origin=OpenAIIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, is_streaming_response) + span = sentry_sdk.traces.start_span( + name=f"chat {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_CHAT, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, + }, + ) _set_completions_api_input_data(span, kwargs, integration) @@ -923,26 +872,15 @@ async def _new_async_chat_completion( # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/completions.py#L585 is_streaming_response = kwargs.get("stream", False) or False - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"chat {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_CHAT, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, - }, - ) - else: - span = get_start_span_function()( - op=consts.OP.GEN_AI_CHAT, - name=f"chat {model}", - origin=OpenAIIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, is_streaming_response) + span = sentry_sdk.traces.start_span( + name=f"chat {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_CHAT, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, + }, + ) _set_completions_api_input_data(span, kwargs, integration) @@ -982,7 +920,7 @@ async def _new_async_chat_completion( def _set_completions_api_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", response: "Union[ChatCompletion, Stream[ChatCompletionChunk], AsyncStream[ChatCompletionChunk]]", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", @@ -1003,7 +941,7 @@ def _set_completions_api_output_data( def _wrap_synchronous_completions_chunk_iterator( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "OpenAIIntegration", start_time: "Optional[float]", messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]", @@ -1022,10 +960,7 @@ def _wrap_synchronous_completions_chunk_iterator( client = sentry_sdk.get_client() for x in old_iterator: - if isinstance(span, StreamedSpan): - span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) with capture_internal_exceptions(): if hasattr(x, "choices") and x.choices is not None: @@ -1074,7 +1009,7 @@ def _wrap_synchronous_completions_chunk_iterator( async def _wrap_asynchronous_completions_chunk_iterator( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "OpenAIIntegration", start_time: "Optional[float]", messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]", @@ -1093,10 +1028,7 @@ async def _wrap_asynchronous_completions_chunk_iterator( client = sentry_sdk.get_client() async for x in old_iterator: - if isinstance(span, StreamedSpan): - span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.model) with capture_internal_exceptions(): if hasattr(x, "choices") and x.choices is not None: @@ -1145,7 +1077,7 @@ async def _wrap_asynchronous_completions_chunk_iterator( def _wrap_synchronous_responses_event_iterator( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "OpenAIIntegration", start_time: "Optional[float]", input: "Optional[Union[str, list[str], ResponseInputParam]]", @@ -1173,10 +1105,7 @@ def _wrap_synchronous_responses_event_iterator( data_buf[0].append(x.delta or "") elif isinstance(x, ResponseCompletedEvent): - if isinstance(span, StreamedSpan): - span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) _calculate_responses_token_usage( input=input, @@ -1218,7 +1147,7 @@ def _wrap_synchronous_responses_event_iterator( async def _wrap_asynchronous_responses_event_iterator( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", integration: "OpenAIIntegration", start_time: "Optional[float]", input: "Optional[Union[str, list[str], ResponseInputParam]]", @@ -1246,10 +1175,7 @@ async def _wrap_asynchronous_responses_event_iterator( data_buf[0].append(x.delta or "") elif isinstance(x, ResponseCompletedEvent): - if isinstance(span, StreamedSpan): - span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) - else: - span.set_data(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) + span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, x.response.model) _calculate_responses_token_usage( input=input, @@ -1291,7 +1217,7 @@ async def _wrap_asynchronous_responses_event_iterator( def _set_responses_api_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", response: "Union[Response, Stream[ResponseStreamEvent], AsyncStream[ResponseStreamEvent]]", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", @@ -1312,7 +1238,7 @@ def _set_responses_api_output_data( def _set_embeddings_output_data( - span: "Union[Span, StreamedSpan]", + span: "StreamedSpan", response: "CreateEmbeddingResponse", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", @@ -1372,52 +1298,29 @@ def _new_sync_embeddings_create(f: "Any", *args: "Any", **kwargs: "Any") -> "Any model = kwargs.get("model") - if has_span_streaming_enabled(client.options): - with sentry_sdk.traces.start_span( - name=f"embeddings {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_EMBEDDINGS, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - }, - ) as span: - _set_embeddings_input_data(span, kwargs, integration) - - try: - response = f(*args, **kwargs) - except Exception as exc: - exc_info = sys.exc_info() - with capture_internal_exceptions(): - _capture_exception(exc) - reraise(*exc_info) - - _set_embeddings_output_data( - span, response, kwargs, integration, finish_span=False - ) + with sentry_sdk.traces.start_span( + name=f"embeddings {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_EMBEDDINGS, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + }, + ) as span: + _set_embeddings_input_data(span, kwargs, integration) - return response - else: - with get_start_span_function()( - op=consts.OP.GEN_AI_EMBEDDINGS, - name=f"embeddings {model}", - origin=OpenAIIntegration.origin, - ) as span: - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - _set_embeddings_input_data(span, kwargs, integration) - - try: - response = f(*args, **kwargs) - except Exception as exc: - exc_info = sys.exc_info() - with capture_internal_exceptions(): - _capture_exception(exc) - reraise(*exc_info) - - _set_embeddings_output_data( - span, response, kwargs, integration, finish_span=False - ) + try: + response = f(*args, **kwargs) + except Exception as exc: + exc_info = sys.exc_info() + with capture_internal_exceptions(): + _capture_exception(exc) + reraise(*exc_info) + + _set_embeddings_output_data( + span, response, kwargs, integration, finish_span=False + ) - return response + return response async def _new_async_embeddings_create( @@ -1432,52 +1335,29 @@ async def _new_async_embeddings_create( model = kwargs.get("model") - if has_span_streaming_enabled(client.options): - with sentry_sdk.traces.start_span( - name=f"embeddings {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_EMBEDDINGS, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - }, - ) as span: - _set_embeddings_input_data(span, kwargs, integration) - - try: - response = await f(*args, **kwargs) - except Exception as exc: - exc_info = sys.exc_info() - with capture_internal_exceptions(): - _capture_exception(exc) - reraise(*exc_info) - - _set_embeddings_output_data( - span, response, kwargs, integration, finish_span=False - ) + with sentry_sdk.traces.start_span( + name=f"embeddings {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_EMBEDDINGS, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + }, + ) as span: + _set_embeddings_input_data(span, kwargs, integration) - return response - else: - with get_start_span_function()( - op=consts.OP.GEN_AI_EMBEDDINGS, - name=f"embeddings {model}", - origin=OpenAIIntegration.origin, - ) as span: - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - _set_embeddings_input_data(span, kwargs, integration) - - try: - response = await f(*args, **kwargs) - except Exception as exc: - exc_info = sys.exc_info() - with capture_internal_exceptions(): - _capture_exception(exc) - reraise(*exc_info) - - _set_embeddings_output_data( - span, response, kwargs, integration, finish_span=False - ) + try: + response = await f(*args, **kwargs) + except Exception as exc: + exc_info = sys.exc_info() + with capture_internal_exceptions(): + _capture_exception(exc) + reraise(*exc_info) + + _set_embeddings_output_data( + span, response, kwargs, integration, finish_span=False + ) - return response + return response def _wrap_embeddings_create(f: "Any") -> "Any": @@ -1519,26 +1399,15 @@ def _new_sync_responses_create( # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 is_streaming_response = kwargs.get("stream", False) or False - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"responses {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_RESPONSES, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, - }, - ) - else: - span = get_start_span_function()( - op=consts.OP.GEN_AI_RESPONSES, - name=f"responses {model}", - origin=OpenAIIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, is_streaming_response) + span = sentry_sdk.traces.start_span( + name=f"responses {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_RESPONSES, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, + }, + ) _set_responses_api_input_data(span, kwargs, integration) @@ -1595,26 +1464,15 @@ async def _new_async_responses_create( # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 is_streaming_response = kwargs.get("stream", False) or False - if has_span_streaming_enabled(client.options): - span = sentry_sdk.traces.start_span( - name=f"responses {model}", - attributes={ - "sentry.op": consts.OP.GEN_AI_RESPONSES, - "sentry.origin": OpenAIIntegration.origin, - SPANDATA.GEN_AI_SYSTEM: "openai", - SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, - }, - ) - else: - span = get_start_span_function()( - op=consts.OP.GEN_AI_RESPONSES, - name=f"responses {model}", - origin=OpenAIIntegration.origin, - ) - span.__enter__() - - span.set_data(SPANDATA.GEN_AI_SYSTEM, "openai") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, is_streaming_response) + span = sentry_sdk.traces.start_span( + name=f"responses {model}", + attributes={ + "sentry.op": consts.OP.GEN_AI_RESPONSES, + "sentry.origin": OpenAIIntegration.origin, + SPANDATA.GEN_AI_SYSTEM: "openai", + SPANDATA.GEN_AI_RESPONSE_STREAMING: is_streaming_response, + }, + ) _set_responses_api_input_data(span, kwargs, integration) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 2701c88c15..c7dfbebf0e 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -63,7 +63,6 @@ from unittest import mock # python 3.3 and above -from sentry_sdk import start_transaction from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.openai import ( OpenAIIntegration, @@ -168,20 +167,16 @@ async def __call__(self, *args, **kwargs): OPENAI_VERSION <= (2, 10, 0), reason="ChatCompletionCustomToolParam is unavailable before.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) def test_chat_completion_tool_definitions( sentry_init, - capture_events, capture_items, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -198,134 +193,68 @@ def test_chat_completion_tool_definitions( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - tools=[ - ChatCompletionFunctionToolParam( - type="function", - function=FunctionDefinition( - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, - }, - strict=True, - ), - ), - ChatCompletionCustomToolParam( - type="custom", - custom=Custom( - name="name", - description="description", - ), - ), - ], - ) - - sentry_sdk.flush() - span = next(item.payload for item in items) - - assert json.loads(span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ - { - "type": "function", - "name": "name", - "description": "description", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + tools=[ + ChatCompletionFunctionToolParam( + type="function", + function=FunctionDefinition( + name="name", + description="description", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, }, - "required": ["city", "state"], - "additionalProperties": False, - }, - }, - { - "type": "custom", - "name": "name", - "description": "description", - }, - ] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - tools=[ - ChatCompletionFunctionToolParam( - type="function", - function=FunctionDefinition( - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, - }, - strict=True, - ), - ), - ChatCompletionCustomToolParam( - type="custom", - custom=Custom( - name="name", - description="description", - ), - ), - ], - ) + strict=True, + ), + ), + ChatCompletionCustomToolParam( + type="custom", + custom=Custom( + name="name", + description="description", + ), + ), + ], + ) - tx = events[0] - assert tx["type"] == "transaction" + sentry_sdk.flush() + span = next(item.payload for item in items) - assert json.loads(tx["spans"][0]["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ - { - "type": "function", - "name": "name", - "description": "description", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, + assert json.loads(span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + { + "type": "function", + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, }, + "required": ["city", "state"], + "additionalProperties": False, }, - { - "type": "custom", - "name": "name", - "description": "description", - }, - ] + }, + { + "type": "custom", + "name": "name", + "description": "description", + }, + ] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -336,20 +265,17 @@ def test_chat_completion_tool_definitions( ) def test_nonstreaming_chat_completion_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -366,95 +292,48 @@ def test_nonstreaming_chat_completion_no_prompts( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content - ) - - assert response == "the model response" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content - ) + response = ( + client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) + .choices[0] + .message.content + ) - assert response == "the model response" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert response == "the model response" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_messages,expected_system_instructions", [ @@ -535,20 +414,17 @@ def test_nonstreaming_chat_completion_no_prompts( ) def test_nonstreaming_chat_completion( sentry_init, - capture_events, capture_items, get_messages, expected_system_instructions, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -565,105 +441,57 @@ def test_nonstreaming_chat_completion( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content - ) - - assert response == "the model response" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions - ) - - assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert ( - "Message demonstrating the absence of truncation." - in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + response = ( + client.chat.completions.create( + model="some-model", + messages=get_messages(), + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) - assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = ( - client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - .choices[0] - .message.content - ) - - assert response == "the model response" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + .choices[0] + .message.content + ) - assert ( - json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions - ) + assert response == "the model response" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + == expected_system_instructions + ) - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the tools parameter.", ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,expected_present,expected_absent", [ @@ -709,21 +537,18 @@ def test_nonstreaming_chat_completion( ) def test_completions_api_data_collection( sentry_init, - capture_events, capture_items, data_collection, expected_present, expected_absent, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -754,25 +579,13 @@ def test_completions_api_data_collection( "top_p": 0.9, "tools": EXAMPLE_COMPLETIONS_TOOLS, } + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create(**create_kwargs) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create(**create_kwargs) + client.chat.completions.create(**create_kwargs) - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "chat" assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" @@ -790,7 +603,6 @@ def test_completions_api_data_collection( assert key not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,expect_output", [ @@ -834,21 +646,18 @@ def test_completions_api_data_collection( ) def test_completions_api_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, expect_output, nonstreaming_chat_completions_model_response, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } if data_collection is not None: init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -869,31 +678,16 @@ def test_completions_api_data_collection_outputs( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" @@ -903,7 +697,6 @@ def test_completions_api_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,expect_output", @@ -948,21 +741,18 @@ def test_completions_api_data_collection_outputs( ) async def test_completions_api_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, expect_output, nonstreaming_chat_completions_model_response, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } if data_collection is not None: init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -983,31 +773,16 @@ async def test_completions_api_data_collection_outputs_async( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-3.5-turbo" @@ -1017,20 +792,16 @@ async def test_completions_api_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) def test_completions_api_data_collection_outputs_empty_choices( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, _experiments={"data_collection": {"gen_ai": {"outputs": True}}}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1048,37 +819,21 @@ def test_completions_api_data_collection_outputs_empty_choices( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - ) + client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + ) - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] # No choices means no output data, even with outputs collection enabled assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,expect_output", [ @@ -1101,13 +856,11 @@ def test_completions_api_data_collection_outputs_empty_choices( ) def test_streaming_chat_completion_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, expect_output, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], @@ -1115,8 +868,7 @@ def test_streaming_chat_completion_data_collection_outputs( traces_sample_rate=1.0, send_default_pii=False, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1140,49 +892,24 @@ def test_streaming_chat_completion_data_collection_outputs( include_event_type=False, ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - - assert response_string == "hello" - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) + with mock.patch.object( + client.chat._client._client, "send", return_value=returned_stream + ): + response_stream = client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + ) + response_string = "".join( + map(lambda x: x.choices[0].delta.content, response_stream) + ) - assert response_string == "hello" - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + assert response_string == "hello" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] if expect_output: assert "hello" in span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -1190,7 +917,6 @@ def test_streaming_chat_completion_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,expect_output", @@ -1214,14 +940,12 @@ def test_streaming_chat_completion_data_collection_outputs( ) async def test_streaming_chat_completion_data_collection_outputs_async( sentry_init, - capture_events, capture_items, data_collection, expect_output, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], @@ -1229,8 +953,7 @@ async def test_streaming_chat_completion_data_collection_outputs_async( traces_sample_rate=1.0, send_default_pii=False, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -1256,49 +979,26 @@ async def test_streaming_chat_completion_data_collection_outputs_async( ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + ) + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - ) - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello" - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - ) - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello" - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + assert response_string == "hello" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] if expect_output: assert "hello" in span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -1306,7 +1006,6 @@ async def test_streaming_chat_completion_data_collection_outputs_async( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -1318,20 +1017,17 @@ async def test_streaming_chat_completion_data_collection_outputs_async( ) async def test_nonstreaming_chat_completion_async_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -1348,89 +1044,45 @@ async def test_nonstreaming_chat_completion_async_no_prompts( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content - - assert response == "the model response" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content + response = await client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) + response = response.choices[0].message.content - assert response == "the model response" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + assert response == "the model response" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "get_messages,expected_system_instructions", @@ -1512,20 +1164,17 @@ async def test_nonstreaming_chat_completion_async_no_prompts( ) async def test_nonstreaming_chat_completion_async( sentry_init, - capture_events, capture_items, get_messages, expected_system_instructions, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -1542,92 +1191,48 @@ async def test_nonstreaming_chat_completion_async( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content - - assert response == "the model response" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions - ) - - assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert ( - "Message demonstrating the absence of truncation." - in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response = response.choices[0].message.content - - assert response == "the model response" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions - ) + response = await client.chat.completions.create( + model="some-model", + messages=get_messages(), + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) + response = response.choices[0].message.content + + assert response == "the model response" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + == expected_system_instructions + ) - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 def tiktoken_encoding_if_installed(): @@ -1640,7 +1245,6 @@ def tiktoken_encoding_if_installed(): # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -1651,13 +1255,11 @@ def tiktoken_encoding_if_installed(): ) def test_streaming_chat_completion_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -1669,8 +1271,7 @@ def test_streaming_chat_completion_no_prompts( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1720,128 +1321,69 @@ def test_streaming_chat_completion_no_prompts( include_event_type=False, ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + stream=True, + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) + response_string = "".join( + map(lambda x: x.choices[0].delta.content, response_stream) + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - - assert response_string == "hello world" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - - assert response_string == "hello world" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert response_string == "hello world" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 7 - assert span["data"]["gen_ai.usage.total_tokens"] == 9 - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the stream_options parameter.", ) def test_streaming_chat_completion_with_usage_in_stream( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """When stream_options=include_usage is set, token usage comes from the final chunk's usage field.""" sentry_init( @@ -1849,8 +1391,7 @@ def test_streaming_chat_completion_with_usage_in_stream( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1892,68 +1433,39 @@ def test_streaming_chat_completion_with_usage_in_stream( include_event_type=False, ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - for _ in response_stream: - pass - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the stream_options parameter.", ) def test_streaming_chat_completion_empty_content_preserves_token_usage( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """Token usage from the stream is recorded even when no content is produced (e.g. content filter).""" sentry_init( @@ -1961,8 +1473,7 @@ def test_streaming_chat_completion_empty_content_preserves_token_usage( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -1985,57 +1496,30 @@ def test_streaming_chat_completion_empty_content_preserves_token_usage( include_event_type=False, ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert "gen_ai.usage.output_tokens" not in span["attributes"] - assert span["attributes"]["gen_ai.usage.total_tokens"] == 20 - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - for _ in response_stream: - pass - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert "gen_ai.usage.output_tokens" not in span["data"] - assert span["data"]["gen_ai.usage.total_tokens"] == 20 + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert "gen_ai.usage.output_tokens" not in span["attributes"] + assert span["attributes"]["gen_ai.usage.total_tokens"] == 20 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the stream_options parameter.", @@ -2043,12 +1527,10 @@ def test_streaming_chat_completion_empty_content_preserves_token_usage( @pytest.mark.asyncio async def test_streaming_chat_completion_empty_content_preserves_token_usage_async( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """Token usage from the stream is recorded even when no content is produced - async variant.""" sentry_init( @@ -2056,8 +1538,7 @@ async def test_streaming_chat_completion_empty_content_preserves_token_usage_asy disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -2082,57 +1563,30 @@ async def test_streaming_chat_completion_empty_content_preserves_token_usage_asy ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + async for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - async for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert "gen_ai.usage.output_tokens" not in span["attributes"] - assert span["attributes"]["gen_ai.usage.total_tokens"] == 20 - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - async for _ in response_stream: - pass - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert "gen_ai.usage.output_tokens" not in span["data"] - assert span["data"]["gen_ai.usage.total_tokens"] == 20 + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert "gen_ai.usage.output_tokens" not in span["attributes"] + assert span["attributes"]["gen_ai.usage.total_tokens"] == 20 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the stream_options parameter.", @@ -2140,12 +1594,10 @@ async def test_streaming_chat_completion_empty_content_preserves_token_usage_asy @pytest.mark.asyncio async def test_streaming_chat_completion_async_with_usage_in_stream( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """When stream_options=include_usage is set, token usage comes from the final chunk's usage field (async).""" sentry_init( @@ -2153,8 +1605,7 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=False, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -2198,58 +1649,31 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + async for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - async for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "hello"}], - stream=True, - stream_options={"include_usage": True}, - ) - async for _ in response_stream: - pass - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_messages,expected_system_instructions,expected_output_tokens,expected_input_tokens", [ @@ -2336,7 +1760,6 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( ) def test_streaming_chat_completion( sentry_init, - capture_events, capture_items, get_messages, expected_system_instructions, @@ -2344,7 +1767,6 @@ def test_streaming_chat_completion( expected_input_tokens, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -2356,8 +1778,7 @@ def test_streaming_chat_completion( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -2407,135 +1828,71 @@ def test_streaming_chat_completion( include_event_type=False, ) ) - - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=get_messages(), - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - assert response_string == "hello world" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = client.chat.completions.create( + model="some-model", + messages=get_messages(), + stream=True, + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert ( - "Message demonstrating the absence of truncation." - in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + response_string = "".join( + map(lambda x: x.choices[0].delta.content, response_stream) ) - assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert response_string == "hello world" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + == expected_system_instructions + ) - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - assert ( - span["attributes"]["gen_ai.usage.output_tokens"] - == expected_output_tokens - ) - assert ( - span["attributes"]["gen_ai.usage.input_tokens"] == expected_input_tokens - ) - assert ( - span["attributes"]["gen_ai.usage.total_tokens"] - == expected_output_tokens + expected_input_tokens - ) + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=get_messages(), - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - response_string = "".join( - map(lambda x: x.choices[0].delta.content, response_stream) - ) - assert response_string == "hello world" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import assert ( - json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions + span["attributes"]["gen_ai.usage.output_tokens"] == expected_output_tokens + ) + assert span["attributes"]["gen_ai.usage.input_tokens"] == expected_input_tokens + assert ( + span["attributes"]["gen_ai.usage.total_tokens"] + == expected_output_tokens + expected_input_tokens ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert span["data"]["gen_ai.usage.output_tokens"] == expected_output_tokens - assert span["data"]["gen_ai.usage.input_tokens"] == expected_input_tokens - assert ( - span["data"]["gen_ai.usage.total_tokens"] - == expected_output_tokens + expected_input_tokens - ) - - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -2547,14 +1904,12 @@ def test_streaming_chat_completion( ) async def test_streaming_chat_completion_async_no_prompts( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -2566,8 +1921,7 @@ async def test_streaming_chat_completion_async_no_prompts( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -2619,122 +1973,63 @@ async def test_streaming_chat_completion_async_no_prompts( ) ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + stream=True, + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello world" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 - - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content - assert response_string == "hello world" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + assert response_string == "hello world" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 7 - assert span["data"]["gen_ai.usage.total_tokens"] == 9 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "get_messages,expected_system_instructions,expected_output_tokens,expected_input_tokens", @@ -2822,7 +2117,6 @@ async def test_streaming_chat_completion_async_no_prompts( ) async def test_streaming_chat_completion_async( sentry_init, - capture_events, capture_items, get_messages, expected_system_instructions, @@ -2831,7 +2125,6 @@ async def test_streaming_chat_completion_async( get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -2843,8 +2136,7 @@ async def test_streaming_chat_completion_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -2892,296 +2184,166 @@ async def test_streaming_chat_completion_async( model="model-id", object="chat.completion.chunk", ), - ], - include_event_type=False, - ) - ) - ) - - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - - assert response_string == "hello world" - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True - - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions + ], + include_event_type=False, + ) ) - - assert ( - "Message demonstrating the absence of truncation." - in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = await client.chat.completions.create( + model="some-model", + messages=get_messages(), + stream=True, + max_tokens=100, + presence_penalty=0.1, + frequency_penalty=0.2, + temperature=0.7, + top_p=0.9, ) - assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert ( - span["attributes"]["gen_ai.usage.output_tokens"] - == expected_output_tokens - ) - assert ( - span["attributes"]["gen_ai.usage.input_tokens"] == expected_input_tokens - ) - assert ( - span["attributes"]["gen_ai.usage.total_tokens"] - == expected_output_tokens + expected_input_tokens - ) - - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=get_messages(), - stream=True, - max_tokens=100, - presence_penalty=0.1, - frequency_penalty=0.2, - temperature=0.7, - top_p=0.9, - ) - - response_string = "" - async for x in response_stream: - response_string += x.choices[0].delta.content - assert response_string == "hello world" - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content + + assert response_string == "hello world" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True + + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + == expected_system_instructions + ) - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "some-model" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY] == 0.1 - assert span["data"][SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY] == 0.2 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) + assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import assert ( - json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - == expected_system_instructions + span["attributes"]["gen_ai.usage.output_tokens"] == expected_output_tokens + ) + assert span["attributes"]["gen_ai.usage.input_tokens"] == expected_input_tokens + assert ( + span["attributes"]["gen_ai.usage.total_tokens"] + == expected_output_tokens + expected_input_tokens ) - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - try: - import tiktoken # type: ignore # noqa # pylint: disable=unused-import - - assert span["data"]["gen_ai.usage.output_tokens"] == expected_output_tokens - assert span["data"]["gen_ai.usage.input_tokens"] == expected_input_tokens - assert ( - span["data"]["gen_ai.usage.total_tokens"] - == expected_output_tokens + expected_input_tokens - ) - - except ImportError: - pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly -@pytest.mark.parametrize("span_streaming", [True, False]) def test_bad_chat_completion( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") - ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) - - (event,) = (item.payload for item in items if item.type == "event") - sentry_sdk.flush() - (span,) = (item.payload for item in items if item.type == "span") - assert event["level"] == "error" - assert span["status"] == "error" - else: - events = capture_events() - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + side_effect=OpenAIError("API rate limit reached") + ) + with pytest.raises(OpenAIError): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items if item.type == "event") + sentry_sdk.flush() + (span,) = (item.payload for item in items if item.type == "span") + assert event["level"] == "error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_status_error( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with start_transaction(name="test"): - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") - ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) - - (error,) = (item.payload for item in items if item.type == "event") - assert error["level"] == "error" - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["status"] == "error" - else: - events = capture_events() + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + side_effect=OpenAIError("API rate limit reached") + ) + with pytest.raises(OpenAIError): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], + ) - with start_transaction(name="test"): - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - side_effect=OpenAIError("API rate limit reached") - ) - with pytest.raises(OpenAIError): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - ) + (error,) = (item.payload for item in items if item.type == "event") + assert error["level"] == "error" - (error, transaction) = events - assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_bad_chat_completion_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") client.chat.completions._post = AsyncMock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with pytest.raises(OpenAIError): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (event,) = (item.payload for item in items if item.type == "event") - sentry_sdk.flush() - (span,) = (item.payload for item in items if item.type == "span") - assert event["level"] == "error" - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(OpenAIError): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) + with pytest.raises(OpenAIError): + await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items if item.type == "event") + sentry_sdk.flush() + (span,) = (item.payload for item in items if item.type == "span") + assert event["level"] == "error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -3192,19 +2354,16 @@ async def test_bad_chat_completion_async( ) def test_embeddings_create_no_pii( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -3220,54 +2379,24 @@ def test_embeddings_create_no_pii( ) client.embeddings._post = mock.Mock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] - == "text-embedding-3-large" - ) - - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + response = client.embeddings.create(input="hello", model="text-embedding-3-large") - assert len(response.data[0].embedding) == 3 + assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["data"] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["attributes"] - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_input,expected_embeddings_input", [ @@ -3339,19 +2468,16 @@ def test_embeddings_create_no_pii( ) def test_embeddings_create( sentry_init, - capture_events, capture_items, get_input, expected_embeddings_input, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -3367,90 +2493,42 @@ def test_embeddings_create( ) client.embeddings._post = mock.Mock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] - == "text-embedding-3-large" - ) - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) - == expected_embeddings_input - ) - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - - assert ( - json.loads(span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) - == expected_embeddings_input - ) - - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + response = client.embeddings.create( + input=get_input(), model="text-embedding-3-large" + ) + assert len(response.data[0].embedding) == 3 -def _collect_embeddings_span_data( - capture_events, capture_items, span_streaming, create -): - if span_streaming: - items = capture_items("span") + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - with start_transaction(name="openai tx"): - response = create() + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) + == expected_embeddings_input + ) - assert len(response.data[0].embedding) == 3 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - return span["attributes"] - events = capture_events() +def _collect_embeddings_span_data(capture_events, capture_items, create): + items = capture_items("span") - with start_transaction(name="openai tx"): - response = create() + response = create() assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - return span["data"] + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + return span["attributes"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,expect_input", [ @@ -3493,15 +2571,13 @@ def test_embeddings_create_data_collection( data_collection, send_default_pii, expect_input, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } sentry_init_kwargs = dict(init_kwargs) @@ -3527,7 +2603,6 @@ def test_embeddings_create_data_collection( span_data = _collect_embeddings_span_data( capture_events, capture_items, - span_streaming, lambda: client.embeddings.create(input="hello", model="text-embedding-3-large"), ) @@ -3544,7 +2619,6 @@ def test_embeddings_create_data_collection( assert span_data["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_input", [ @@ -3561,16 +2635,14 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( capture_events, capture_items, get_input, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", + trace_lifecycle="stream", _experiments={"data_collection": {"gen_ai": {"inputs": False}}}, - stream_gen_ai_spans=False, ) client = OpenAI(api_key="z") @@ -3590,7 +2662,6 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( span_data = _collect_embeddings_span_data( capture_events, capture_items, - span_streaming, lambda: client.embeddings.create( input=get_input(), model="text-embedding-3-large" ), @@ -3601,7 +2672,6 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -3613,19 +2683,16 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( ) async def test_embeddings_create_async_no_pii( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -3641,54 +2708,26 @@ async def test_embeddings_create_async_no_pii( ) client.embeddings._post = AsyncMock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] - == "text-embedding-3-large" - ) - - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) - assert len(response.data[0].embedding) == 3 + assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["data"] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in span["attributes"] - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "get_input,expected_embeddings_input", @@ -3761,19 +2800,16 @@ async def test_embeddings_create_async_no_pii( ) async def test_embeddings_create_async( sentry_init, - capture_events, capture_items, get_input, expected_embeddings_input, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -3789,60 +2825,29 @@ async def test_embeddings_create_async( ) client.embeddings._post = AsyncMock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] - == "text-embedding-3-large" - ) - - assert ( - json.loads(span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) - == expected_embeddings_input - ) - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input=get_input(), model="text-embedding-3-large" - ) + response = await client.embeddings.create( + input=get_input(), model="text-embedding-3-large" + ) - assert len(response.data[0].embedding) == 3 + assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "text-embedding-3-large" - assert ( - json.loads(span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) - == expected_embeddings_input - ) + assert ( + json.loads(span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT]) + == expected_embeddings_input + ) - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "data_collection,send_default_pii,expect_input", @@ -3875,20 +2880,17 @@ async def test_embeddings_create_async( ) async def test_embeddings_create_async_data_collection( sentry_init, - capture_events, capture_items, data_collection, send_default_pii, expect_input, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } sentry_init_kwargs = dict(init_kwargs) @@ -3910,36 +2912,18 @@ async def test_embeddings_create_async_data_collection( ) client.embeddings._post = AsyncMock(return_value=returned_embedding) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - assert len(response.data[0].embedding) == 3 - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" - span_data = span["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response = await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + response = await client.embeddings.create( + input="hello", model="text-embedding-3-large" + ) - assert len(response.data[0].embedding) == 3 + assert len(response.data[0].embedding) == 3 - tx = events[0] - assert tx["type"] == "transaction" - span = tx["spans"][0] - assert span["op"] == "gen_ai.embeddings" - span_data = span["data"] + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.embeddings" + span_data = span["attributes"] assert span_data[SPANDATA.GEN_AI_SYSTEM] == "openai" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "embeddings" @@ -3954,26 +2938,22 @@ async def test_embeddings_create_async_data_collection( assert span_data["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [(True, True), (True, False), (False, True), (False, False)], ) def test_embeddings_create_raises_error( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -3981,30 +2961,18 @@ def test_embeddings_create_raises_error( client.embeddings._post = mock.Mock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with pytest.raises(OpenAIError): - client.embeddings.create(input="hello", model="text-embedding-3-large") - - (event,) = (item.payload for item in items if item.type == "event") - sentry_sdk.flush() - (span,) = (item.payload for item in items if item.type == "span") - assert event["level"] == "error" - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(OpenAIError): - client.embeddings.create(input="hello", model="text-embedding-3-large") + with pytest.raises(OpenAIError): + client.embeddings.create(input="hello", model="text-embedding-3-large") - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items if item.type == "event") + sentry_sdk.flush() + (span,) = (item.payload for item in items if item.type == "span") + assert event["level"] == "error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -4012,19 +2980,16 @@ def test_embeddings_create_raises_error( ) async def test_embeddings_create_raises_error_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -4032,46 +2997,27 @@ async def test_embeddings_create_raises_error_async( client.embeddings._post = AsyncMock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with pytest.raises(OpenAIError): - await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - (event,) = (item.payload for item in items if item.type == "event") - sentry_sdk.flush() - (span,) = (item.payload for item in items if item.type == "span") - assert event["level"] == "error" - assert span["status"] == "error" - else: - events = capture_events() - - with pytest.raises(OpenAIError): - await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) + with pytest.raises(OpenAIError): + await client.embeddings.create(input="hello", model="text-embedding-3-large") - (event, transaction) = events - assert event["level"] == "error" - assert transaction["contexts"]["trace"]["status"] == "internal_error" + (event,) = (item.payload for item in items if item.type == "event") + sentry_sdk.flush() + (span,) = (item.payload for item in items if item.type == "span") + assert event["level"] == "error" + assert span["status"] == "error" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_origin_nonstreaming_chat( sentry_init, - capture_events, capture_items, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -4088,48 +3034,28 @@ def test_span_origin_nonstreaming_chat( ), ) ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (event,) = events + client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_origin_nonstreaming_chat_async( sentry_init, - capture_events, capture_items, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -4146,46 +3072,26 @@ async def test_span_origin_nonstreaming_chat_async( ), ) ) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - (event,) = events + await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_origin_streaming_chat( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -4225,54 +3131,31 @@ def test_span_origin_streaming_chat( object="chat.completion.chunk", ), ] + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - client.chat.completions._post = mock.Mock(return_value=returned_stream) - with sentry_sdk.traces.start_span(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - "".join(map(lambda x: x.choices[0].delta.content, response_stream)) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - client.chat.completions._post = mock.Mock(return_value=returned_stream) - with start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - - "".join(map(lambda x: x.choices[0].delta.content, response_stream)) + client.chat.completions._post = mock.Mock(return_value=returned_stream) + response_stream = client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) - (event,) = events + "".join(map(lambda x: x.choices[0].delta.content, response_stream)) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_origin_streaming_chat_async( sentry_init, - capture_events, capture_items, async_iterator, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -4318,54 +3201,30 @@ async def test_span_origin_streaming_chat_async( ) client.chat.completions._post = AsyncMock(return_value=returned_stream) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - async for _ in response_stream: - pass - - # "".join(map(lambda x: x.choices[0].delta.content, response_stream)) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] - ) - async for _ in response_stream: - pass - - # "".join(map(lambda x: x.choices[0].delta.content, response_stream)) + response_stream = await client.chat.completions.create( + model="some-model", messages=[{"role": "system", "content": "hello"}] + ) + async for _ in response_stream: + pass - (event,) = events + # "".join(map(lambda x: x.choices[0].delta.content, response_stream)) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" -@pytest.mark.parametrize("span_streaming", [True, False]) def test_span_origin_embeddings( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -4381,43 +3240,25 @@ def test_span_origin_embeddings( ) client.embeddings._post = mock.Mock(return_value=returned_embedding) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.embeddings.create(input="hello", model="text-embedding-3-large") - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.embeddings.create(input="hello", model="text-embedding-3-large") + client.embeddings.create(input="hello", model="text-embedding-3-large") - (event,) = events + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_span_origin_embeddings_async( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -4433,31 +3274,13 @@ async def test_span_origin_embeddings_async( ) client.embeddings._post = AsyncMock(return_value=returned_embedding) + items = capture_items("transaction", "span") - if span_streaming: - items = capture_items("transaction", "span") - - with sentry_sdk.traces.start_span(name="openai tx"): - await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" - else: - events = capture_events() - - with start_transaction(name="openai tx"): - await client.embeddings.create( - input="hello", model="text-embedding-3-large" - ) - - (event,) = events + await client.embeddings.create(input="hello", model="text-embedding-3-large") - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.ai.openai" + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.origin"] == "auto.ai.openai" def test_completions_token_usage_from_response(): @@ -4822,228 +3645,88 @@ def count_tokens(msg): ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_ai_client_span_responses_api_no_pii( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) - - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - - assert len(spans) == 2 - expected_attributes = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.system": "openai", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "sentry.op": "gen_ai.responses", - "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", - } - - for attr, value in expected_attributes.items(): - assert spans[0]["attributes"][attr] == value - - assert "gen_ai.system_instructions" not in spans[0]["attributes"] - assert "gen_ai.request.messages" not in spans[0]["attributes"] - assert "gen_ai.response.text" not in spans[0]["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - assert spans[0]["origin"] == "auto.ai.openai" - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.system": "openai", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - } - - for key, value in expected_data.items(): - assert spans[0]["data"][key] == value - - assert "gen_ai.system_instructions" not in spans[0]["data"] - assert "gen_ai.request.messages" not in spans[0]["data"] - assert "gen_ai.response.text" not in spans[0]["data"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) -@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -def test_ai_client_span_responses_tool_definitions( - sentry_init, - capture_events, - capture_items, - span_streaming, -): - sentry_init( - integrations=[OpenAIIntegration()], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + items = capture_items("span") + + client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, ) - client = OpenAI(api_key="z") - client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) - - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="How do I check if a Python object is an instance of a class?", - tools=[ - FunctionToolParam( - type="function", - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, - }, - strict=True, - ), - CustomToolParam( - type="custom", name="name", description="description" - ), - WebSearchToolParam(type="web_search"), - ], - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - assert json.loads(spans[0]["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ - { - "type": "function", - "name": "name", - "description": "description", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, - }, - }, - { - "type": "custom", - "name": "name", - "description": "description", - }, - { - "type": "web_search", - }, - ] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="How do I check if a Python object is an instance of a class?", - tools=[ - FunctionToolParam( - type="function", - name="name", - description="description", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "state": {"type": "string"}, - }, - "required": ["city", "state"], - "additionalProperties": False, - }, - strict=True, - ), - CustomToolParam( - type="custom", name="name", description="description" - ), - WebSearchToolParam(type="web_search"), - ], - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + + assert len(spans) == 1 + expected_attributes = { + "gen_ai.operation.name": "responses", + "gen_ai.request.max_tokens": 100, + "gen_ai.request.temperature": 0.7, + "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.model": "response-model-id", + "gen_ai.response.streaming": False, + "gen_ai.system": "openai", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "sentry.op": "gen_ai.responses", + "sentry.origin": "auto.ai.openai", + } - (transaction,) = events - spans = transaction["spans"] + for attr, value in expected_attributes.items(): + assert spans[0]["attributes"][attr] == value - assert json.loads(spans[0]["data"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ - { - "type": "function", - "name": "name", - "description": "description", - "parameters": { + assert "gen_ai.system_instructions" not in spans[0]["attributes"] + assert "gen_ai.request.messages" not in spans[0]["attributes"] + assert "gen_ai.response.text" not in spans[0]["attributes"] + + +@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") +def test_ai_client_span_responses_tool_definitions( + sentry_init, + capture_items, +): + sentry_init( + integrations=[OpenAIIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + trace_lifecycle="stream", + ) + + client = OpenAI(api_key="z") + client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") + + client.responses.create( + model="gpt-4o", + input="How do I check if a Python object is an instance of a class?", + tools=[ + FunctionToolParam( + type="function", + name="name", + description="description", + parameters={ "type": "object", "properties": { "city": {"type": "string"}, @@ -5052,19 +3735,41 @@ def test_ai_client_span_responses_tool_definitions( "required": ["city", "state"], "additionalProperties": False, }, + strict=True, + ), + CustomToolParam(type="custom", name="name", description="description"), + WebSearchToolParam(type="web_search"), + ], + ) + + sentry_sdk.flush() + spans = [item.payload for item in items] + assert json.loads(spans[0]["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS]) == [ + { + "type": "function", + "name": "name", + "description": "description", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "state": {"type": "string"}, + }, + "required": ["city", "state"], + "additionalProperties": False, }, - { - "type": "custom", - "name": "name", - "description": "description", - }, - { - "type": "web_search", - }, - ] + }, + { + "type": "custom", + "name": "name", + "description": "description", + }, + { + "type": "web_search", + }, + ] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "instructions,input,expected_system_instructions,expected_request_messages", [ @@ -5211,125 +3916,69 @@ def test_ai_client_span_responses_tool_definitions( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_ai_client_span_responses_api( sentry_init, - capture_events, capture_items, instructions, input, expected_system_instructions, expected_request_messages, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") + + client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - - assert len(spans) == 2 - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.system": "openai", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.request.messages": safe_serialize(expected_request_messages), - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.text": "the model response", - "sentry.op": "gen_ai.responses", - "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) - - for attr, value in expected_data.items(): - assert spans[0]["attributes"][attr] == value - - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + + assert len(spans) == 1 + + expected_data = { + "gen_ai.operation.name": "responses", + "gen_ai.request.max_tokens": 100, + "gen_ai.request.temperature": 0.7, + "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", + "gen_ai.system": "openai", + "gen_ai.response.model": "response-model-id", + "gen_ai.response.streaming": False, + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.request.messages": safe_serialize(expected_request_messages), + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.text": "the model response", + "sentry.op": "gen_ai.responses", + "sentry.origin": "auto.ai.openai", + } - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.system": "openai", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.request.messages": safe_serialize(expected_request_messages[-1:]), - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.text": "the model response", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) + if expected_system_instructions is not None: + expected_data["gen_ai.system_instructions"] = safe_serialize( + expected_system_instructions + ) - for attr, value in expected_data.items(): - assert spans[0]["data"][attr] == value + for attr, value in expected_data.items(): + assert spans[0]["attributes"][attr] == value -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,extra_kwargs,expected_present,expected_absent,include_prompts", [ @@ -5456,22 +4105,19 @@ def test_ai_client_span_responses_api( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_responses_api_data_collection( sentry_init, - capture_events, capture_items, data_collection, extra_kwargs, expected_present, expected_absent, include_prompts, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -5485,30 +4131,15 @@ def test_responses_api_data_collection( "reasoning": {"effort": "high"}, } create_kwargs.update(extra_kwargs) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - client.responses.create(**create_kwargs) + client.responses.create(**create_kwargs) - sentry_sdk.flush() - spans = [item.payload for item in items] + sentry_sdk.flush() + spans = [item.payload for item in items] - assert len(spans) == 2 - span_data = spans[0]["attributes"] - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create(**create_kwargs) - - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - span_data = spans[0]["data"] + assert len(spans) == 1 + span_data = spans[0]["attributes"] # Non-input data is always collected, regardless of data collection config assert span_data["gen_ai.operation.name"] == "responses" @@ -5571,29 +4202,16 @@ def _make_responses_api_response(output): ) -def _collect_responses_span_data(capture_events, capture_items, span_streaming, create): - if span_streaming: - items = capture_items("span") +def _collect_responses_span_data(capture_events, capture_items, create): + items = capture_items("span") - with start_transaction(name="openai tx"): - create() + create() - sentry_sdk.flush() - (span,) = (item.payload for item in items) - return span["attributes"] + sentry_sdk.flush() + (span,) = (item.payload for item in items) + return span["attributes"] - events = capture_events() - with start_transaction(name="openai tx"): - create() - - (transaction,) = events - (span,) = transaction["spans"] - assert span["op"] == "gen_ai.responses" - return span["data"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,send_default_pii,expect_output", [ @@ -5637,15 +4255,13 @@ def test_responses_api_data_collection_outputs( data_collection, send_default_pii, expect_output, - span_streaming, ): init_kwargs = { "integrations": [OpenAIIntegration()], "disabled_integrations": [StdlibIntegration], "traces_sample_rate": 1.0, "send_default_pii": send_default_pii, - "trace_lifecycle": "stream" if span_streaming else "static", - "stream_gen_ai_spans": False, + "trace_lifecycle": "stream", } if data_collection is not None: init_kwargs["_experiments"] = {"data_collection": data_collection} @@ -5673,7 +4289,6 @@ def test_responses_api_data_collection_outputs( span_data = _collect_responses_span_data( capture_events, capture_items, - span_streaming, lambda: client.responses.create(model="gpt-4o", input="hello"), ) @@ -5687,7 +4302,6 @@ def test_responses_api_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "get_output,expect_text,expect_tool_calls", [ @@ -5738,15 +4352,13 @@ def test_responses_api_data_collection_outputs_shapes( get_output, expect_text, expect_tool_calls, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, _experiments={"data_collection": {"gen_ai": {"outputs": True}}}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -5757,7 +4369,6 @@ def test_responses_api_data_collection_outputs_shapes( span_data = _collect_responses_span_data( capture_events, capture_items, - span_streaming, lambda: client.responses.create(model="gpt-4o", input="hello"), ) @@ -5772,7 +4383,6 @@ def test_responses_api_data_collection_outputs_shapes( assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "data_collection,expect_output", [ @@ -5796,13 +4406,11 @@ def test_responses_api_data_collection_outputs_shapes( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_streaming_responses_api_data_collection_outputs( sentry_init, - capture_events, capture_items, data_collection, expect_output, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], @@ -5810,8 +4418,7 @@ def test_streaming_responses_api_data_collection_outputs( traces_sample_rate=1.0, send_default_pii=False, _experiments={"data_collection": data_collection}, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -5820,51 +4427,27 @@ def test_streaming_responses_api_data_collection_outputs( EXAMPLE_RESPONSES_STREAM, ) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ): + response_stream = client.responses.create( + model="some-model", + input="hello", + stream=True, + ) + response_string = "" + for item in response_stream: + if hasattr(item, "delta"): + response_string += item.delta - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - response_string = "" - for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta - - assert response_string == "hello world" - sentry_sdk.flush() - (span,) = (item.payload for item in items) - span_data = span["attributes"] - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - response_string = "" - for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta - - assert response_string == "hello world" - (transaction,) = events - (span,) = transaction["spans"] - span_data = span["data"] + assert response_string == "hello world" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + span_data = span["attributes"] if expect_output: assert "hello world" in span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -5872,7 +4455,6 @@ def test_streaming_responses_api_data_collection_outputs( assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "conversation, expected_id", [ @@ -5885,60 +4467,36 @@ def test_streaming_responses_api_data_collection_outputs( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_responses_api_conversation_id( sentry_init, - capture_events, capture_items, conversation, expected_id, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - conversation=conversation, - ) + client.responses.create( + model="gpt-4o", + input="hello", + conversation=conversation, + ) - sentry_sdk.flush() - (span,) = (item.payload for item in items) + sentry_sdk.flush() + (span,) = (item.payload for item in items) - if expected_id is None: - assert "gen_ai.conversation.id" not in span["attributes"] - else: - assert span["attributes"]["gen_ai.conversation.id"] == expected_id + if expected_id is None: + assert "gen_ai.conversation.id" not in span["attributes"] else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - conversation=conversation, - ) - - (transaction,) = events - (span,) = transaction["spans"] + assert span["attributes"]["gen_ai.conversation.id"] == expected_id - if expected_id is None: - assert "gen_ai.conversation.id" not in span["data"] - else: - assert span["data"]["gen_ai.conversation.id"] == expected_id - -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "reasoning, expected_level", [ @@ -5951,134 +4509,78 @@ def test_responses_api_conversation_id( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_responses_api_reasoning_level( sentry_init, - capture_events, capture_items, reasoning, expected_level, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - reasoning=reasoning, - ) - - sentry_sdk.flush() - span = next(item.payload for item in items if item.type == "span") + client.responses.create( + model="gpt-4o", + input="hello", + reasoning=reasoning, + ) - if expected_level is None: - assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] - else: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] - == expected_level - ) + sentry_sdk.flush() + span = next(item.payload for item in items if item.type == "span") + if expected_level is None: + assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.responses.create( - model="gpt-4o", - input="hello", - reasoning=reasoning, - ) - - (transaction,) = events - span = transaction["spans"][0] - - if expected_level is None: - assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["data"] - else: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == expected_level - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] + == expected_level + ) -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_error_in_responses_api( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") client.responses._post = mock.Mock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with sentry_sdk.traces.start_span(name="openai tx"), pytest.raises(OpenAIError): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) - - # make sure the span where the error occurred is captured - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["attributes"]["sentry.op"] == "gen_ai.responses" - - (error_event,) = (item.payload for item in items if item.type == "event") - - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - - assert spans[1]["is_segment"] is True - assert error_event["contexts"]["trace"]["trace_id"] == spans[1]["trace_id"] - else: - events = capture_events() + with pytest.raises(OpenAIError): + client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) - with start_transaction(name="openai tx"), pytest.raises(OpenAIError): - client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) + # make sure the span where the error occurred is captured + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.op"] == "gen_ai.responses" - (error_event, transaction_event) = events + (error_event,) = (item.payload for item in items if item.type == "event") - assert transaction_event["type"] == "transaction" - # make sure the span where the error occurred is captured - assert transaction_event["spans"][0]["op"] == "gen_ai.responses" + assert error_event["level"] == "error" + assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert ( - error_event["contexts"]["trace"]["trace_id"] - == transaction_event["contexts"]["trace"]["trace_id"] - ) + assert error_event["contexts"]["trace"]["trace_id"] == spans[0]["trace_id"] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") @pytest.mark.parametrize( @@ -6226,125 +4728,69 @@ def test_error_in_responses_api( ) async def test_ai_client_span_responses_async_api( sentry_init, - capture_events, capture_items, instructions, input, expected_system_instructions, expected_request_messages, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") client.responses._post = AsyncMock(return_value=EXAMPLE_RESPONSE) + items = capture_items("span") + + await client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="openai tx"): - await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - sentry_sdk.flush() - spans = [item.payload for item in items] - - assert len(spans) == 2 - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.request.messages": safe_serialize(expected_request_messages), - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.system": "openai", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - "sentry.op": "gen_ai.responses", - "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) - - for attr, value in expected_data.items(): - assert spans[0]["attributes"][attr] == value - - else: - events = capture_events() - - with start_transaction(name="openai tx"): - await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) + sentry_sdk.flush() + spans = [item.payload for item in items] + + assert len(spans) == 1 + + expected_data = { + "gen_ai.operation.name": "responses", + "gen_ai.request.max_tokens": 100, + "gen_ai.request.temperature": 0.7, + "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", + "gen_ai.request.messages": safe_serialize(expected_request_messages), + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.model": "response-model-id", + "gen_ai.response.streaming": False, + "gen_ai.system": "openai", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.response.text": "the model response", + "sentry.op": "gen_ai.responses", + "sentry.origin": "auto.ai.openai", + } - (transaction,) = events - spans = transaction["spans"] - - assert len(spans) == 1 - assert spans[0]["op"] == "gen_ai.responses" - assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.request.messages": safe_serialize(expected_request_messages[-1:]), - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": False, - "gen_ai.system": "openai", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) + if expected_system_instructions is not None: + expected_data["gen_ai.system_instructions"] = safe_serialize( + expected_system_instructions + ) - for attr, value in expected_data.items(): - assert spans[0]["data"][attr] == value + for attr, value in expected_data.items(): + assert spans[0]["attributes"][attr] == value -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "instructions,input,expected_system_instructions,expected_request_messages", @@ -6492,7 +4938,6 @@ async def test_ai_client_span_responses_async_api( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_ai_client_span_streaming_responses_async_api( sentry_init, - capture_events, capture_items, instructions, input, @@ -6501,215 +4946,119 @@ async def test_ai_client_span_streaming_responses_async_api( get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") returned_stream = get_model_response( async_iterator(server_side_event_chunks(EXAMPLE_RESPONSES_STREAM)) ) - - if span_streaming: - items = capture_items("span") - - ctx = ( - sentry_sdk.traces.start_span(name="openai tx") - if span_streaming - else start_transaction(name="openai tx") + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ): + result = await client.responses.create( + model="gpt-4o", + instructions=instructions, + input=input, + stream=True, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, ) - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), ctx: - result = await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - async for _ in result: - pass - - sentry_sdk.flush() - spans = [item.payload for item in items] - spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == OP.GEN_AI_RESPONSES - ] - - assert len(spans) == 1 - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.messages": safe_serialize(expected_request_messages), - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": True, - "gen_ai.system": "openai", - "gen_ai.response.time_to_first_token": mock.ANY, - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.text": "hello world", - "sentry.environment": "production", - "sentry.op": "gen_ai.responses", - "sentry.origin": "auto.ai.openai", - "sentry.segment.name": "openai tx", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) - - for attr, value in expected_data.items(): - assert spans[0]["attributes"][attr] == value - - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - result = await client.responses.create( - model="gpt-4o", - instructions=instructions, - input=input, - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - async for _ in result: - pass + async for _ in result: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items] + spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == OP.GEN_AI_RESPONSES + ] - (transaction,) = events - spans = [ - span for span in transaction["spans"] if span["op"] == OP.GEN_AI_RESPONSES - ] + assert len(spans) == 1 + + expected_data = { + "gen_ai.operation.name": "responses", + "gen_ai.request.max_tokens": 100, + "gen_ai.request.messages": safe_serialize(expected_request_messages), + "gen_ai.request.temperature": 0.7, + "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", + "gen_ai.response.model": "response-model-id", + "gen_ai.response.streaming": True, + "gen_ai.system": "openai", + "gen_ai.response.time_to_first_token": mock.ANY, + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.text": "hello world", + "sentry.environment": "production", + "sentry.op": "gen_ai.responses", + "sentry.origin": "auto.ai.openai", + } - assert len(spans) == 1 - assert spans[0]["origin"] == "auto.ai.openai" - - expected_data = { - "gen_ai.operation.name": "responses", - "gen_ai.request.max_tokens": 100, - "gen_ai.request.messages": safe_serialize(expected_request_messages[-1:]), - "gen_ai.request.temperature": 0.7, - "gen_ai.request.top_p": 0.9, - "gen_ai.request.reasoning.level": "high", - "gen_ai.response.model": "response-model-id", - "gen_ai.response.streaming": True, - "gen_ai.system": "openai", - "gen_ai.response.time_to_first_token": mock.ANY, - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.request.model": "gpt-4o", - "gen_ai.response.text": "hello world", - } - - if expected_system_instructions is not None: - expected_data["gen_ai.system_instructions"] = safe_serialize( - expected_system_instructions - ) + if expected_system_instructions is not None: + expected_data["gen_ai.system_instructions"] = safe_serialize( + expected_system_instructions + ) - for attr, value in expected_data.items(): - assert spans[0]["data"][attr] == value + for attr, value in expected_data.items(): + assert spans[0]["attributes"][attr] == value -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_error_in_responses_async_api( sentry_init, - capture_events, capture_items, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") client.responses._post = AsyncMock( side_effect=OpenAIError("API rate limit reached") ) + items = capture_items("event", "span") - if span_streaming: - items = capture_items("event", "span") - - with sentry_sdk.traces.start_span(name="openai tx"), pytest.raises(OpenAIError): - await client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) - - # make sure the span where the error occurred is captured - sentry_sdk.flush() - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["attributes"]["sentry.op"] == "gen_ai.responses" - - (error_event,) = (item.payload for item in items if item.type == "event") - - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - - assert spans[1]["is_segment"] is True - assert error_event["contexts"]["trace"]["trace_id"] == spans[1]["trace_id"] - else: - events = capture_events() + with pytest.raises(OpenAIError): + await client.responses.create( + model="gpt-4o", + instructions="You are a coding assistant that talks like a pirate.", + input="How do I check if a Python object is an instance of a class?", + ) - with start_transaction(name="openai tx"), pytest.raises(OpenAIError): - await client.responses.create( - model="gpt-4o", - instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", - ) + # make sure the span where the error occurred is captured + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["attributes"]["sentry.op"] == "gen_ai.responses" - (error_event, transaction_event) = events + (error_event,) = (item.payload for item in items if item.type == "event") - assert transaction_event["type"] == "transaction" - # make sure the span where the error occurred is captured - assert transaction_event["spans"][0]["op"] == "gen_ai.responses" + assert error_event["level"] == "error" + assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert error_event["level"] == "error" - assert error_event["exception"]["values"][0]["type"] == "OpenAIError" - assert ( - error_event["contexts"]["trace"]["trace_id"] - == transaction_event["contexts"]["trace"]["trace_id"] - ) + assert error_event["contexts"]["trace"]["trace_id"] == spans[0]["trace_id"] if SKIP_RESPONSES_TESTS: @@ -6786,7 +5135,6 @@ async def test_error_in_responses_async_api( ] -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "send_default_pii, include_prompts", [(True, True), (True, False), (False, True), (False, False)], @@ -6794,13 +5142,11 @@ async def test_error_in_responses_async_api( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_streaming_responses_api( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -6811,8 +5157,7 @@ def test_streaming_responses_api( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -6821,102 +5166,53 @@ def test_streaming_responses_api( EXAMPLE_RESPONSES_STREAM, ) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ): + response_stream = client.responses.create( + model="some-model", + input="hello", + stream=True, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - response_string = "" - for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta - - assert response_string == "hello world" - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.responses" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" - - if send_default_pii and include_prompts: - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - response_string = "" - for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta + response_string = "" + for item in response_stream: + if hasattr(item, "delta"): + response_string += item.delta - assert response_string == "hello world" + assert response_string == "hello world" - (transaction,) = events - (span,) = transaction["spans"] - assert span["op"] == "gen_ai.responses" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.responses" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" - if send_default_pii and include_prompts: - assert span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -6925,14 +5221,12 @@ def test_streaming_responses_api( @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_streaming_responses_api_async( sentry_init, - capture_events, capture_items, send_default_pii, include_prompts, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): sentry_init( integrations=[ @@ -6943,110 +5237,60 @@ async def test_streaming_responses_api_async( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=send_default_pii, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") returned_stream = get_model_response( async_iterator(server_side_event_chunks(EXAMPLE_RESPONSES_STREAM)) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ): + response_stream = await client.responses.create( + model="some-model", + input="hello", + stream=True, + max_output_tokens=100, + temperature=0.7, + top_p=0.9, + reasoning={"effort": "high"}, + ) - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.responses.create( - model="some-model", - input="hello", - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - response_string = "" - async for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta - - assert response_string == "hello world" - - sentry_sdk.flush() - (span,) = (item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.responses" - assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" - - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" - - if send_default_pii and include_prompts: - assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - - assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 - assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.responses.create( - model="some-model", - input="hello", - stream=True, - max_output_tokens=100, - temperature=0.7, - top_p=0.9, - reasoning={"effort": "high"}, - ) - - response_string = "" - async for item in response_stream: - if hasattr(item, "delta"): - response_string += item.delta + response_string = "" + async for item in response_stream: + if hasattr(item, "delta"): + response_string += item.delta - assert response_string == "hello world" + assert response_string == "hello world" - (transaction,) = events - (span,) = transaction["spans"] - assert span["op"] == "gen_ai.responses" - assert span["data"][SPANDATA.GEN_AI_SYSTEM] == "openai" - assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 - assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 - assert span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" + sentry_sdk.flush() + (span,) = (item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.responses" + assert span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "openai" + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" - if send_default_pii and include_prompts: - assert span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + if send_default_pii and include_prompts: + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '["hello"]' + assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "hello world" + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] - assert span["data"]["gen_ai.usage.input_tokens"] == 20 - assert span["data"]["gen_ai.usage.output_tokens"] == 10 - assert span["data"]["gen_ai.usage.total_tokens"] == 30 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.parametrize("span_streaming", [True, False]) # Feature added in https://github.com/openai/openai-python/pull/1952 @pytest.mark.skipif( OPENAI_VERSION is None or OPENAI_VERSION < (1, 58, 0), @@ -7063,19 +5307,16 @@ async def test_streaming_responses_api_async( ) def test_chat_completion_reasoning_level( sentry_init, - capture_events, capture_items, reasoning_effort, expected_level, nonstreaming_chat_completions_model_response, - span_streaming, ): sentry_init( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -7092,50 +5333,27 @@ def test_chat_completion_reasoning_level( ), ) ) + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - reasoning_effort=reasoning_effort, - ) + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], + reasoning_effort=reasoning_effort, + ) - sentry_sdk.flush() - span = next(item.payload for item in items if item.type == "span") + sentry_sdk.flush() + span = next(item.payload for item in items if item.type == "span") - if expected_level is None: - assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] - else: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] - == expected_level - ) + if expected_level is None: + assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - reasoning_effort=reasoning_effort, - ) - - (event,) = events - span = event["spans"][0] - - if expected_level is None: - assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["data"] - else: - assert ( - span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == expected_level - ) + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] + == expected_level + ) # Test messages with mixed roles including "ai" that should be mapped to "assistant" -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize( "test_message,expected_role", [ @@ -7152,12 +5370,10 @@ def test_chat_completion_reasoning_level( ) def test_openai_message_role_mapping( sentry_init, - capture_events, capture_items, test_message, expected_role, nonstreaming_chat_completions_model_response, - span_streaming, ): """Test that OpenAI integration properly maps message roles like 'ai' to 'assistant'""" @@ -7166,8 +5382,7 @@ def test_openai_message_role_mapping( disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -7186,112 +5401,28 @@ def test_openai_message_role_mapping( ) test_messages = [test_message] + items = capture_items("span") - if span_streaming: - items = capture_items("span") - - with start_transaction(name="openai tx"): - client.chat.completions.create(model="test-model", messages=test_messages) - - # Verify that the span was created correctly - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create(model="test-model", messages=test_messages) + client.chat.completions.create(model="test-model", messages=test_messages) - # Verify that the span was created correctly - (event,) = events - span = event["spans"][0] - assert span["op"] == "gen_ai.chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] + # Verify that the span was created correctly + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + stored_messages = json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert len(stored_messages) == 1 assert stored_messages[0]["role"] == expected_role -def test_openai_message_truncation( - sentry_init, - capture_events, - nonstreaming_chat_completions_model_response, -): - """Test that large messages are truncated properly in OpenAI integration.""" - sentry_init( - integrations=[OpenAIIntegration(include_prompts=True)], - traces_sample_rate=1.0, - send_default_pii=True, - stream_gen_ai_spans=False, - ) - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock( - return_value=nonstreaming_chat_completions_model_response( - response_id="chat-id", - response_model="gpt-3.5-turbo", - message_content="the model response", - created=10000000, - usage=CompletionUsage( - prompt_tokens=20, - completion_tokens=10, - total_tokens=30, - ), - ) - ) - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - large_messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": large_content}, - {"role": "assistant", "content": large_content}, - {"role": "user", "content": large_content}, - ] - - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=large_messages, - ) - - (event,) = events - span = event["spans"][0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - - messages_data = span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) <= len(large_messages) - - meta_path = event["_meta"] - span_meta = meta_path["spans"]["0"]["data"] - messages_meta = span_meta[SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "len" in messages_meta.get("", {}) - - # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_chat_completion_ttft( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """ Test that streaming chat completions capture time-to-first-token (TTFT). @@ -7300,8 +5431,7 @@ def test_streaming_chat_completion_ttft( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") @@ -7338,71 +5468,42 @@ def test_streaming_chat_completion_ttft( include_event_type=False, ), ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + ) + # Consume the stream + for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "Say hello"}], - stream=True, - ) - # Consume the stream - for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] - ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "Say hello"}], - stream=True, - ) - # Consume the stream - for _ in response_stream: - pass - - (tx,) = events - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["data"] - ttft = span["data"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] + # Verify TTFT is captured + assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] + ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] assert isinstance(ttft, float) assert ttft > 0 # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio async def test_streaming_chat_completion_ttft_async( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """ Test that async streaming chat completions capture time-to-first-token (TTFT). @@ -7411,8 +5512,7 @@ async def test_streaming_chat_completion_ttft_async( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") @@ -7451,70 +5551,41 @@ async def test_streaming_chat_completion_ttft_async( ), ) ) + items = capture_items("span") + + with mock.patch.object( + client.chat._client._client, + "send", + return_value=returned_stream, + ): + response_stream = await client.chat.completions.create( + model="some-model", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + ) + # Consume the stream + async for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "Say hello"}], - stream=True, - ) - # Consume the stream - async for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.chat" - - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] - ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] - else: - events = capture_events() - - with mock.patch.object( - client.chat._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.chat.completions.create( - model="some-model", - messages=[{"role": "user", "content": "Say hello"}], - stream=True, - ) - # Consume the stream - async for _ in response_stream: - pass - - (tx,) = events - span = tx["spans"][0] - assert span["op"] == "gen_ai.chat" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.chat" - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["data"] - ttft = span["data"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] + # Verify TTFT is captured + assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] + ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] assert isinstance(ttft, float) assert ttft > 0 # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") def test_streaming_responses_api_ttft( sentry_init, - capture_events, capture_items, get_model_response, server_side_event_chunks, - span_streaming, ): """ Test that streaming responses API captures time-to-first-token (TTFT). @@ -7523,80 +5594,50 @@ def test_streaming_responses_api_ttft( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = OpenAI(api_key="z") returned_stream = get_model_response( server_side_event_chunks(EXAMPLE_RESPONSES_STREAM) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ): + response_stream = client.responses.create( + model="some-model", + input="hello", + stream=True, + ) + # Consume the stream + for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - # Consume the stream - for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.responses" - - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] - ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - # Consume the stream - for _ in response_stream: - pass - - (tx,) = events - span = tx["spans"][0] - assert span["op"] == "gen_ai.responses" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.responses" - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["data"] - ttft = span["data"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] + # Verify TTFT is captured + assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] + ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] assert isinstance(ttft, float) assert ttft > 0 # noinspection PyTypeChecker -@pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.asyncio @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") async def test_streaming_responses_api_ttft_async( sentry_init, - capture_events, capture_items, get_model_response, async_iterator, server_side_event_chunks, - span_streaming, ): """ Test that async streaming responses API captures time-to-first-token (TTFT). @@ -7605,63 +5646,36 @@ async def test_streaming_responses_api_ttft_async( integrations=[OpenAIIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - stream_gen_ai_spans=False, + trace_lifecycle="stream", ) client = AsyncOpenAI(api_key="z") returned_stream = get_model_response( async_iterator(server_side_event_chunks(EXAMPLE_RESPONSES_STREAM)) ) + items = capture_items("span") + + with mock.patch.object( + client.responses._client._client, + "send", + return_value=returned_stream, + ): + response_stream = await client.responses.create( + model="some-model", + input="hello", + stream=True, + ) + # Consume the stream + async for _ in response_stream: + pass - if span_streaming: - items = capture_items("span") - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - # Consume the stream - async for _ in response_stream: - pass - - sentry_sdk.flush() - span = next(item.payload for item in items) - assert span["attributes"]["sentry.op"] == "gen_ai.responses" - - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] - ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] - else: - events = capture_events() - - with mock.patch.object( - client.responses._client._client, - "send", - return_value=returned_stream, - ), start_transaction(name="openai tx"): - response_stream = await client.responses.create( - model="some-model", - input="hello", - stream=True, - ) - # Consume the stream - async for _ in response_stream: - pass - - (tx,) = events - span = tx["spans"][0] - assert span["op"] == "gen_ai.responses" + sentry_sdk.flush() + span = next(item.payload for item in items) + assert span["attributes"]["sentry.op"] == "gen_ai.responses" - # Verify TTFT is captured - assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["data"] - ttft = span["data"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] + # Verify TTFT is captured + assert SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN in span["attributes"] + ttft = span["attributes"][SPANDATA.GEN_AI_RESPONSE_TIME_TO_FIRST_TOKEN] assert isinstance(ttft, float) assert ttft > 0