diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index e8920c4e845b..3fc8edf15566 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -20,7 +20,9 @@ import { import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { isOrchestrionEnabled } from '../../../../utils'; -describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { +const expectedOrigin = isOrchestrionEnabled() ? 'auto.vercelai.channel' : 'auto.vercelai.otel'; + +describe('Vercel AI integration (v5)', () => { afterAll(() => { cleanupChildProcesses(); }); @@ -47,6 +49,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); expect(firstInvokeAgentSpan!.status).toBe('ok'); expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); @@ -122,6 +125,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); @@ -267,6 +271,9 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { let errorEvent: Event | undefined; await createRunner() + // The tool error is captured while the tool is running (mid-transaction), so the error event + // and the transaction/span envelopes can arrive in either order — assert content, not wire order. + .unordered() .expect({ transaction: transaction => { transactionEvent = transaction; @@ -291,6 +298,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { expect(toolSpan!.name).toBe('execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); }, }) diff --git a/dev-packages/node-integration-tests/utils/runner/createEsmAndCjsTests.ts b/dev-packages/node-integration-tests/utils/runner/createEsmAndCjsTests.ts index 5def48851504..91de78c7c5ab 100644 --- a/dev-packages/node-integration-tests/utils/runner/createEsmAndCjsTests.ts +++ b/dev-packages/node-integration-tests/utils/runner/createEsmAndCjsTests.ts @@ -366,10 +366,15 @@ const NPM_INSTALL_RETRY_DELAY_MS = 2_000; async function npmInstallWithRetry(cwd: string, deps: string[]): Promise { for (let attempt = 1; attempt <= NPM_INSTALL_MAX_RETRIES; attempt++) { try { - const { stdout, stderr } = await execPromise('npm install --prefer-offline --silent --no-audit --no-fund', { - cwd, - encoding: 'utf8', - }); + const { stdout, stderr } = await execPromise( + // We only use --prefer-offline on the first attempt to try to read from cache + // in follow ups we just try to install in any way + `npm install ${attempt === 1 ? '--prefer-offline' : ''} --no-audit --no-fund`, + { + cwd, + encoding: 'utf8', + }, + ); if (process.env.DEBUG) { // eslint-disable-next-line no-console @@ -388,8 +393,10 @@ async function npmInstallWithRetry(cwd: string, deps: string[]): Promise { ); await new Promise(resolve => setTimeout(resolve, NPM_INSTALL_RETRY_DELAY_MS)); } else { + // oxlint-disable-next-line no-console + console.error(error); throw new Error( - `Failed to install additionalDependencies in tmp dir ${cwd} after ${NPM_INSTALL_MAX_RETRIES} attempts: ${error}`, + `Failed to install additionalDependencies in tmp dir ${cwd} after ${NPM_INSTALL_MAX_RETRIES} attempts`, ); } } diff --git a/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts b/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts index 28fea1cb5229..17a8acac7a30 100644 --- a/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts +++ b/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts @@ -2,7 +2,7 @@ import type { IntegrationFn } from '@sentry/core'; import { defineIntegration, extendIntegration, waitForTracingChannelBinding } from '@sentry/core'; import { vercelAiIntegration as baseVercelAiIntegration } from '../../vercel-ai'; import * as dc from 'node:diagnostics_channel'; -import { subscribeVercelAiOrchestrionChannels } from '../../vercel-ai/vercel-ai-orchestrion-v6-subscriber'; +import { subscribeVercelAiOrchestrionChannels } from '../../vercel-ai/vercel-ai-orchestrion-subscriber'; type VercelAiOptions = Parameters[0]; @@ -35,6 +35,6 @@ const _vercelAiChannelIntegration = ((options: VercelAiOptions = {}) => { /** * Auto-instrument the `ai` SDK. Supported are: * - v7 via native `ai:telemetry` tracing channel - * - v6 via orchestrion `orchestrion:ai:*` channels + * - v5 & v6 via orchestrion `orchestrion:ai:*` channels */ export const vercelAiChannelIntegration = defineIntegration(_vercelAiChannelIntegration); diff --git a/packages/server-utils/src/orchestrion/config/vercel-ai.ts b/packages/server-utils/src/orchestrion/config/vercel-ai.ts index bf1be42796f8..d2be04f4984f 100644 --- a/packages/server-utils/src/orchestrion/config/vercel-ai.ts +++ b/packages/server-utils/src/orchestrion/config/vercel-ai.ts @@ -9,18 +9,22 @@ export const vercelAiConfig = [ // `streamText` returns its result synchronously (streaming is lazy), so it's // `Sync`; the subscriber binds the span via `bindTracingChannelToSpan`, which // ends it when the (synchronous) call returns. - ...vercelAiV6Entries('generateText', 'generateText', 'Async'), - ...vercelAiV6Entries('streamText', 'streamText', 'Sync'), - ...vercelAiV6Entries('embed', 'embed', 'Async'), - ...vercelAiV6Entries('embedMany', 'embedMany', 'Async'), - ...vercelAiV6Entries('executeToolCall', 'executeToolCall', 'Async'), - ...vercelAiV6Entries('resolveLanguageModel', 'resolveLanguageModel', 'Sync'), + // Vercel AI v5: same top-level entry points as v6 + ...vercelAiEntries('>=5.0.0 <7.0.0', 'generateText', 'generateText', 'Async'), + ...vercelAiEntries('>=5.0.0 <7.0.0', 'streamText', 'streamText', 'Sync'), + ...vercelAiEntries('>=5.0.0 <7.0.0', 'embed', 'embed', 'Async'), + ...vercelAiEntries('>=5.0.0 <7.0.0', 'embedMany', 'embedMany', 'Async'), + ...vercelAiEntries('>=5.0.0 <7.0.0', 'resolveLanguageModel', 'resolveLanguageModel', 'Sync'), + // This only exists in >= v6 + ...vercelAiEntries('>=6.0.0 <7.0.0', 'executeToolCall', 'executeToolCall', 'Async'), ] satisfies InstrumentationConfig[]; export const vercelAiChannels = { - // Vercel AI (`ai`) v6: orchestrion injects these so the same channel-based + // Vercel AI (`ai`) v5 & v6: orchestrion injects these so the same channel-based // integration that consumes `ai`'s native `ai:telemetry` channel (v7) can - // also instrument v6. Each maps to a top-level function in `ai`'s bundle. + // also instrument v5/v6. Each maps to a top-level function in `ai`'s bundle. + // v5 and v6 share the same channel names (the subscriber is version-agnostic); + // `VERCEL_AI_EXECUTE_TOOL_CALL` is v6-only (v5 has no `executeToolCall` export). VERCEL_AI_GENERATE_TEXT: 'orchestrion:ai:generateText', VERCEL_AI_STREAM_TEXT: 'orchestrion:ai:streamText', VERCEL_AI_EMBED: 'orchestrion:ai:embed', @@ -49,10 +53,15 @@ export const vercelAiChannels = { * function needs one config entry per file (the app loads whichever matches its * module system). This expands a single target into both. */ -function vercelAiV6Entries(channelName: string, functionName: string, kind: 'Async' | 'Sync'): InstrumentationConfig[] { +function vercelAiEntries( + versionRange: string, + channelName: string, + functionName: string, + kind: 'Async' | 'Sync', +): InstrumentationConfig[] { return ['dist/index.js', 'dist/index.mjs'].map(filePath => ({ channelName, - module: { name: 'ai', versionRange: '>=6.0.0 <7.0.0', filePath }, + module: { name: 'ai', versionRange, filePath }, functionQuery: { functionName, kind }, })); } diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index 1f25d48edff4..da71fd98ea4e 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -530,7 +530,7 @@ function partsFromTextAndToolCalls(text: unknown, toolCalls: unknown): Array= 7 publishes a normalized `ai:telemetry` tracing channel natively - * (consumed by `subscribeVercelAiTracingChannel`). v6 has no such channel, so + * (consumed by `subscribeVercelAiTracingChannel`). v5/v6 have no such channel, so * orchestrion injects `orchestrion:ai:*` channels around the top-level * functions (see `orchestrion/config/index.ts`). The injected channels carry only the * wrapped call's `{ arguments, result, error }` — NOT v7's normalized `event` * object — so this adapter reconstructs an equivalent {@link VercelAiChannelMessage} - * from v6's argument/result shapes and delegates to the SAME span-building core + * from v5/v6's argument/result shapes and delegates to the SAME span-building core * (`createSpanFromMessage` / `enrichSpanOnEnd`) the v7 subscriber uses, so the - * emitted spans are identical between v6 and v7. + * emitted spans are identical between v5, v6 and v7. * * The model call (`languageModelCall` / `generate_content` span) has no * injectable definition in `ai`, so we instead wrap `resolveLanguageModel` (the - * single chokepoint every model call flows through) and monkey-patch - * `doGenerate`/`doStream` on the returned model. + * single chokepoint every model call flows through, present in both v5 and v6) + * and monkey-patch `doGenerate`/`doStream` on the returned model. + * + * Tool-call spans differ by version: v6 exposes a per-call `executeToolCall` + * function orchestrion wraps into its own channel. v5 has no such export (only a + * batch `executeTools`), so instead we monkey-patch each tool's `execute` from + * the operation's `tools`. On v6 that patch is inert: `executeToolCall` runs + * `execute` inside its own tool-call span (the active async context), so the + * patch sees that span as its parent and skips — only v5 (where the parent is + * the enclosing `invoke_agent` span) emits the patched span, so v6 never + * double-counts tool spans. */ /** Shape orchestrion's transform attaches to the tracing-channel context. */ @@ -51,10 +62,14 @@ interface ResolvedModel { } const PATCHED = Symbol('SentryVercelAiModelPatched'); +const TOOL_PATCHED = Symbol('SentryVercelAiToolPatched'); /** A resolved model with our patch bookkeeping (idempotency flag). */ type PatchableModel = ResolvedModel & { [PATCHED]?: boolean }; +/** A tool definition off an operation's `tools`, with our patch bookkeeping. */ +type PatchableTool = { execute?: (...args: unknown[]) => unknown; [TOOL_PATCHED]?: boolean }; + // Per-operation correlation id. No Date/random (unavailable / non-deterministic) — a counter is enough. let callIdCounter = 0; function nextCallId(): string { @@ -68,6 +83,10 @@ const messages = new WeakMap(); // parent against this set (so it never mis-attributes to the enclosing `main`/user span) and reads the // parent's `callId` so its span can be named after the operation (e.g. `ai.streamText.doStream`). const operationSpans = new WeakSet(); +// Tool-call spans — only v6's `executeToolCall` channel opens these. The v5 tool-`execute` patch +// consults this to avoid double-counting: `executeToolCall` runs `execute` inside its span (which is +// therefore the active parent), so a resolved parent in this set means v6 already spanned the call. +const toolCallSpans = new WeakSet(); const callIdBySpan = new WeakMap(); // The operation's per-call recording flags (`experimental_telemetry.recordInputs/recordOutputs`), keyed by // its span. A model call carries no telemetry of its own, so it inherits the enclosing operation's flags — @@ -198,11 +217,21 @@ function bindOperation( if (span) { messages.set(data, message); operationSpans.add(span); + // v6's `executeToolCall` span: tracked so the v5 tool-`execute` patch can recognize (and skip) + // tool calls it runs, since that patch sees this span as its active parent (see `patchToolExecute`). + if (message.type === 'executeTool') { + toolCallSpans.add(span); + } const callId = asString(message.event.callId); if (callId) { callIdBySpan.set(span, callId); } recordingBySpan.set(span, recording(telemetry)); + // v5 has no `executeToolCall` channel, so patch each tool's `execute` to emit the tool-call span. + // Inert on v6 (guarded inside `patchToolExecute` when the parent is `executeToolCall`'s own span). + if (isRecord(callOptions.tools)) { + patchOperationTools(callOptions.tools, options); + } } return span; }; @@ -394,6 +423,92 @@ function patchModelMethod( }; } +/** + * Patch every executable tool on an operation's `tools` (a record keyed by tool name) so each + * invocation emits a `gen_ai.execute_tool` span. v5 routes tool execution through the batch + * `executeTools`, which binds `tool.execute` at call time — so replacing the `execute` property here + * (before the call runs) is picked up. Tools without an `execute` (client-side tools) are skipped, + * matching `executeTools`. + */ +function patchOperationTools(tools: Record, options: VercelAiChannelOptions): void { + // This runs inside the channel `start` transform (no upstream try/catch), so a throw here would break + // the user's `ai` call. Instrumentation must never do that — degrade to no tool span instead. + try { + for (const [toolName, tool] of Object.entries(tools)) { + if (isRecord(tool)) { + patchToolExecute(toolName, tool as PatchableTool, tools, options); + } + } + } catch { + DEBUG_BUILD && debug.log('Vercel AI orchestrion tool patching failed.'); + } +} + +function patchToolExecute( + toolName: string, + tool: PatchableTool, + tools: Record, + options: VercelAiChannelOptions, +): void { + const original = tool.execute; + if (typeof original !== 'function' || tool[TOOL_PATCHED]) { + return; + } + tool[TOOL_PATCHED] = true; + tool.execute = function (this: unknown, input: unknown, ...rest: unknown[]): unknown { + // Skip if there's no enclosing operation span (telemetry disabled for the call), or if that parent + // is itself a tool-call span — the latter means v6's `executeToolCall` already opened a span for + // this call and is running `execute` inside it, so spanning again here would double-count. On v5 + // the parent is the enclosing `invoke_agent` operation span, so we proceed. + const parent = resolveModelCallParent(); + if (!parent || toolCallSpans.has(parent)) { + return original.apply(this, [input, ...rest]); + } + + // v5 passes `{ toolCallId, messages, abortSignal, ... }` as the second argument to `execute`. + const callOptions = isRecord(rest[0]) ? rest[0] : {}; + const message: VercelAiChannelMessage = { + type: 'executeTool', + event: { + callId: callIdBySpan.get(parent), + toolCall: { toolName, toolCallId: asString(callOptions.toolCallId), input }, + // The `tools` record (keyed by name) lets the shared core backfill the tool's `description`. + tools, + // Inherit the enclosing operation's per-call recording flags so tool inputs/outputs are recorded + // whenever they are on the parent `invoke_agent` span. + ...recordingBySpan.get(parent), + }, + }; + const span = withActiveSpan(parent, () => createSpanFromMessage(message, options)); + // `executeTool` always opens a span; the guard just keeps the wrapper safe if that changes. + if (!span) { + return original.apply(this, [input, ...rest]); + } + + // v5's `executeTools` catches a thrown tool error and turns it into `tool-error` content rather + // than rejecting, so the user never sees a rejection — we must capture it here. Rethrow so + // `executeTools` still produces its `tool-error` result and the operation continues normally. + const failSpan = (error: unknown): never => { + captureToolError(span, message, error); + span.end(); + throw error; + }; + + try { + const result = Promise.resolve(original.apply(this, [input, ...rest])); + return result.then(value => { + // The shared core (matching v6/v7) expects the tool result nested under `output`. + message.result = { output: value }; + enrichSpanOnEnd(span, message, options); + span.end(); + return value; + }, failSpan); + } catch (error) { + return failSpan(error); + } + }; +} + function buildTextMessage(type: 'generateText' | 'streamText'): MessageBuilder { return (options, telemetry) => ({ type, @@ -422,7 +537,15 @@ function normalizePromptMessages(options: Record): unknown { } function recording(telemetry: Record): { recordInputs: unknown; recordOutputs: unknown } { - return { recordInputs: telemetry.recordInputs, recordOutputs: telemetry.recordOutputs }; + // Match the OTel integration's per-call default: an explicit `recordInputs`/`recordOutputs` wins, but a + // call that merely enables telemetry (`isEnabled: true`, no explicit flags) defaults both to `true`. + // Unlike v7's native `ai:telemetry` channel, the orchestrion channels expose `isEnabled`, so — as noted + // in `resolveRecording` — we CAN reproduce that default here (the native-channel subscriber cannot). + const enabledDefault = telemetry.isEnabled === true ? true : undefined; + return { + recordInputs: telemetry.recordInputs ?? enabledDefault, + recordOutputs: telemetry.recordOutputs ?? enabledDefault, + }; } function modelFields(model: unknown): { provider?: string; modelId?: string } {