From 87ed0608417b7da872471a611996a04a327f5322 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 31 Mar 2026 14:14:46 +0200 Subject: [PATCH 1/5] feat(langchain): Record run_name in on_chat_model_start --- sentry_sdk/integrations/langchain.py | 7 +++ .../integrations/langchain/test_langchain.py | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index c786b22562..2d64c22325 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -432,6 +432,13 @@ def on_chat_model_start( SPANDATA.GEN_AI_AGENT_NAME, agent_metadata["lc_agent_name"] ) + run_name = kwargs.get("name") + if run_name is not None: + span.set_data( + SPANDATA.GEN_AI_PIPELINE_NAME, + run_name, + ) + for key, attribute in DATA_FIELDS.items(): if key in all_params and all_params[key] is not None: set_data_normalized(span, attribute, all_params[key], unpack=False) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 9243fcda53..0f8b5bed51 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -170,6 +170,56 @@ def test_langchain_text_completion( assert llm_span["data"]["gen_ai.usage.output_tokens"] == 15 +def test_langchain_chat( + sentry_init, + capture_events, + get_model_response, + nonstreaming_responses_model_response, +): + sentry_init( + integrations=[ + LangchainIntegration( + include_prompts=True, + ) + ], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + model_response = get_model_response( + nonstreaming_responses_model_response, + serialize_pydantic=True, + request_headers={ + "X-Stainless-Raw-Response": "True", + }, + ) + + llm = ChatOpenAI( + model_name="gpt-3.5-turbo", + temperature=0, + openai_api_key="badkey", + use_responses_api=True, + ) + + with patch.object( + llm.client._client._client, + "send", + return_value=model_response, + ) as _: + with start_transaction(): + llm.invoke( + "How many letters in the word eudca", + config={"run_name": "my-snazzy-pipeline"}, + ) + + tx = events[0] + + chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") + assert len(chat_spans) == 1 + assert chat_spans[0]["data"]["gen_ai.pipeline.name"] == "my-snazzy-pipeline" + + @pytest.mark.skipif( LANGCHAIN_VERSION < (1,), reason="LangChain 1.0+ required (ONE AGENT refactor)", From ea94bfcb1241ee3110fb657f182c4f194466c424 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 31 Mar 2026 14:37:34 +0200 Subject: [PATCH 2/5] . --- tests/conftest.py | 24 ++++ .../integrations/langchain/test_langchain.py | 5 +- tests/integrations/openai/test_openai.py | 109 +++++++++++------- 3 files changed, 94 insertions(+), 44 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 71f2431aac..6a15d3668f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1102,6 +1102,30 @@ def nonstreaming_responses_model_response(): ) +@pytest.fixture +def nonstreaming_chat_completions_model_response(): + return openai.types.chat.ChatCompletion( + id="chat-id", + choices=[ + openai.types.chat.chat_completion.Choice( + index=0, + finish_reason="stop", + message=openai.types.chat.ChatCompletionMessage( + role="assistant", content="the model response" + ), + ) + ], + created=10000000, + model="response-model-id", + object="chat.completion", + usage=openai.types.CompletionUsage( + completion_tokens=10, + prompt_tokens=20, + total_tokens=30, + ), + ) + + @pytest.fixture def responses_tool_call_model_responses(): def inner( diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 0f8b5bed51..269b4052e2 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -174,7 +174,7 @@ def test_langchain_chat( sentry_init, capture_events, get_model_response, - nonstreaming_responses_model_response, + nonstreaming_chat_completions_model_response, ): sentry_init( integrations=[ @@ -188,7 +188,7 @@ def test_langchain_chat( events = capture_events() model_response = get_model_response( - nonstreaming_responses_model_response, + nonstreaming_chat_completions_model_response, serialize_pydantic=True, request_headers={ "X-Stainless-Raw-Response": "True", @@ -199,7 +199,6 @@ def test_langchain_chat( model_name="gpt-3.5-turbo", temperature=0, openai_api_key="badkey", - use_responses_api=True, ) with patch.object( diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 0fd049e742..cd60afe551 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -15,9 +15,8 @@ Omit = None from openai import AsyncOpenAI, OpenAI, AsyncStream, Stream, OpenAIError -from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding -from openai.types.chat import ChatCompletion, ChatCompletionMessage, ChatCompletionChunk -from openai.types.chat.chat_completion import Choice +from openai.types import CreateEmbeddingResponse, Embedding +from openai.types.chat import ChatCompletionChunk from openai.types.chat.chat_completion_chunk import ChoiceDelta, Choice as DeltaChoice from openai.types.create_embedding_response import Usage as EmbeddingTokenUsage @@ -60,26 +59,6 @@ async def __call__(self, *args, **kwargs): OPENAI_VERSION = package_version("openai") -EXAMPLE_CHAT_COMPLETION = ChatCompletion( - id="chat-id", - choices=[ - Choice( - index=0, - finish_reason="stop", - message=ChatCompletionMessage( - role="assistant", content="the model response" - ), - ) - ], - created=10000000, - model="response-model-id", - object="chat.completion", - usage=CompletionUsage( - completion_tokens=10, - prompt_tokens=20, - total_tokens=30, - ), -) if SKIP_RESPONSES_TESTS: @@ -131,7 +110,11 @@ async def __call__(self, *args, **kwargs): ], ) def test_nonstreaming_chat_completion_no_prompts( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, + capture_events, + send_default_pii, + include_prompts, + nonstreaming_chat_completions_model_response, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], @@ -141,7 +124,9 @@ def test_nonstreaming_chat_completion_no_prompts( events = capture_events() client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response + ) with start_transaction(name="openai tx"): response = ( @@ -228,7 +213,13 @@ def test_nonstreaming_chat_completion_no_prompts( ), ], ) -def test_nonstreaming_chat_completion(sentry_init, capture_events, messages, request): +def test_nonstreaming_chat_completion( + sentry_init, + capture_events, + messages, + request, + nonstreaming_chat_completions_model_response, +): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, @@ -237,7 +228,9 @@ def test_nonstreaming_chat_completion(sentry_init, capture_events, messages, req events = capture_events() client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response + ) with start_transaction(name="openai tx"): response = ( @@ -307,7 +300,11 @@ def test_nonstreaming_chat_completion(sentry_init, capture_events, messages, req ], ) async def test_nonstreaming_chat_completion_async_no_prompts( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, + capture_events, + send_default_pii, + include_prompts, + nonstreaming_chat_completions_model_response, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=include_prompts)], @@ -317,7 +314,9 @@ async def test_nonstreaming_chat_completion_async_no_prompts( events = capture_events() client = AsyncOpenAI(api_key="z") - client.chat.completions._post = mock.AsyncMock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = mock.AsyncMock( + return_value=nonstreaming_chat_completions_model_response + ) with start_transaction(name="openai tx"): response = await client.chat.completions.create( @@ -403,7 +402,11 @@ async def test_nonstreaming_chat_completion_async_no_prompts( ], ) async def test_nonstreaming_chat_completion_async( - sentry_init, capture_events, messages, request + sentry_init, + capture_events, + messages, + request, + nonstreaming_chat_completions_model_response, ): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], @@ -413,7 +416,9 @@ async def test_nonstreaming_chat_completion_async( events = capture_events() client = AsyncOpenAI(api_key="z") - client.chat.completions._post = AsyncMock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = AsyncMock( + return_value=nonstreaming_chat_completions_model_response + ) with start_transaction(name="openai tx"): response = await client.chat.completions.create( @@ -1551,7 +1556,9 @@ async def test_embeddings_create_raises_error_async( assert event["level"] == "error" -def test_span_origin_nonstreaming_chat(sentry_init, capture_events): +def test_span_origin_nonstreaming_chat( + sentry_init, capture_events, nonstreaming_chat_completions_model_response +): sentry_init( integrations=[OpenAIIntegration()], traces_sample_rate=1.0, @@ -1559,7 +1566,9 @@ def test_span_origin_nonstreaming_chat(sentry_init, capture_events): events = capture_events() client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response + ) with start_transaction(name="openai tx"): client.chat.completions.create( @@ -1573,7 +1582,9 @@ def test_span_origin_nonstreaming_chat(sentry_init, capture_events): @pytest.mark.asyncio -async def test_span_origin_nonstreaming_chat_async(sentry_init, capture_events): +async def test_span_origin_nonstreaming_chat_async( + sentry_init, capture_events, nonstreaming_chat_completions_model_response +): sentry_init( integrations=[OpenAIIntegration()], traces_sample_rate=1.0, @@ -1581,7 +1592,9 @@ async def test_span_origin_nonstreaming_chat_async(sentry_init, capture_events): events = capture_events() client = AsyncOpenAI(api_key="z") - client.chat.completions._post = AsyncMock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = AsyncMock( + return_value=nonstreaming_chat_completions_model_response + ) with start_transaction(name="openai tx"): await client.chat.completions.create( @@ -3125,7 +3138,9 @@ async def test_streaming_responses_api_async( "tools", [[], None, NOT_GIVEN, omit], ) -def test_empty_tools_in_chat_completion(sentry_init, capture_events, tools): +def test_empty_tools_in_chat_completion( + sentry_init, capture_events, tools, nonstreaming_chat_completions_model_response +): sentry_init( integrations=[OpenAIIntegration()], traces_sample_rate=1.0, @@ -3133,7 +3148,9 @@ def test_empty_tools_in_chat_completion(sentry_init, capture_events, tools): events = capture_events() client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response + ) with start_transaction(name="openai tx"): client.chat.completions.create( @@ -3164,7 +3181,11 @@ def test_empty_tools_in_chat_completion(sentry_init, capture_events, tools): ], ) def test_openai_message_role_mapping( - sentry_init, capture_events, test_message, expected_role + sentry_init, + capture_events, + test_message, + expected_role, + nonstreaming_chat_completions_model_response, ): """Test that OpenAI integration properly maps message roles like 'ai' to 'assistant'""" @@ -3176,7 +3197,9 @@ def test_openai_message_role_mapping( events = capture_events() client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response + ) test_messages = [test_message] @@ -3197,7 +3220,9 @@ def test_openai_message_role_mapping( assert stored_messages[0]["role"] == expected_role -def test_openai_message_truncation(sentry_init, capture_events): +def test_openai_message_truncation( + sentry_init, capture_events, nonstreaming_chat_completions_model_response +): """Test that large messages are truncated properly in OpenAI integration.""" sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], @@ -3207,7 +3232,9 @@ def test_openai_message_truncation(sentry_init, capture_events): events = capture_events() client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response + ) large_content = ( "This is a very long message that will exceed our size limits. " * 1000 From cd08d96d0cfd7c9c900808ec55649a8eeca13225 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 31 Mar 2026 15:06:21 +0200 Subject: [PATCH 3/5] . --- tests/integrations/langchain/test_langchain.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 269b4052e2..e514b30958 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -67,6 +67,7 @@ ) LANGCHAIN_VERSION = package_version("langchain") +LANGCHAIN_OPENAI_VERSION = package_version("langchain-openai") @tool @@ -187,12 +188,15 @@ def test_langchain_chat( ) events = capture_events() + request_headers = {} + # Changed in https://github.com/langchain-ai/langchain/pull/32655 + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 32): + request_headers["X-Stainless-Raw-Response"] = "True" + model_response = get_model_response( nonstreaming_chat_completions_model_response, serialize_pydantic=True, - request_headers={ - "X-Stainless-Raw-Response": "True", - }, + request_headers=request_headers, ) llm = ChatOpenAI( From 568e6f7eb4858423793369536b53ec8310ee4a69 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 31 Mar 2026 15:24:59 +0200 Subject: [PATCH 4/5] truthy check --- sentry_sdk/integrations/langchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 2d64c22325..8115b660b1 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -433,7 +433,7 @@ def on_chat_model_start( ) run_name = kwargs.get("name") - if run_name is not None: + if run_name: span.set_data( SPANDATA.GEN_AI_PIPELINE_NAME, run_name, From 52eb5c358d8e318920d3ccdd60ceea69e8881505 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 1 Apr 2026 11:28:44 +0200 Subject: [PATCH 5/5] more descriptive test name --- tests/integrations/langchain/test_langchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index e514b30958..bd9ab894c4 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -171,7 +171,7 @@ def test_langchain_text_completion( assert llm_span["data"]["gen_ai.usage.output_tokens"] == 15 -def test_langchain_chat( +def test_langchain_chat_with_run_name( sentry_init, capture_events, get_model_response,