From 5e45df268ca17236af9c1c6d48c5e2edf171ca60 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Mon, 6 Jul 2026 13:29:31 +0200 Subject: [PATCH 1/5] WIP WIP --- .../suites/tracing/vercelai/v5/test.ts | 10 +- .../integrations/tracing-channel/vercel-ai.ts | 4 +- .../src/orchestrion/config/vercel-ai.ts | 37 +++-- .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 2 +- ...ts => vercel-ai-orchestrion-subscriber.ts} | 155 ++++++++++++++++-- 5 files changed, 176 insertions(+), 32 deletions(-) rename packages/server-utils/src/vercel-ai/{vercel-ai-orchestrion-v6-subscriber.ts => vercel-ai-orchestrion-subscriber.ts} (73%) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index e8920c4e845b..ed0c8b73114a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -20,7 +20,12 @@ import { import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { isOrchestrionEnabled } from '../../../../utils'; -describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { +// In orchestrion mode the channel-based integration emits the spans (origin `auto.vercelai.channel`); +// otherwise the OTel-based integration does (`auto.vercelai.otel`). The spans are otherwise identical, +// so the same assertions run in both modes — only the origin differs. +const expectedOrigin = isOrchestrionEnabled() ? 'auto.vercelai.channel' : 'auto.vercelai.otel'; + +describe('Vercel AI integration (v5)', () => { afterAll(() => { cleanupChildProcesses(); }); @@ -47,6 +52,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); expect(firstInvokeAgentSpan!.status).toBe('ok'); expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); @@ -122,6 +128,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); @@ -291,6 +298,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (v5)', () => { expect(toolSpan!.name).toBe('execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); }, }) diff --git a/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts b/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts index 28fea1cb5229..17a8acac7a30 100644 --- a/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts +++ b/packages/server-utils/src/integrations/tracing-channel/vercel-ai.ts @@ -2,7 +2,7 @@ import type { IntegrationFn } from '@sentry/core'; import { defineIntegration, extendIntegration, waitForTracingChannelBinding } from '@sentry/core'; import { vercelAiIntegration as baseVercelAiIntegration } from '../../vercel-ai'; import * as dc from 'node:diagnostics_channel'; -import { subscribeVercelAiOrchestrionChannels } from '../../vercel-ai/vercel-ai-orchestrion-v6-subscriber'; +import { subscribeVercelAiOrchestrionChannels } from '../../vercel-ai/vercel-ai-orchestrion-subscriber'; type VercelAiOptions = Parameters[0]; @@ -35,6 +35,6 @@ const _vercelAiChannelIntegration = ((options: VercelAiOptions = {}) => { /** * Auto-instrument the `ai` SDK. Supported are: * - v7 via native `ai:telemetry` tracing channel - * - v6 via orchestrion `orchestrion:ai:*` channels + * - v5 & v6 via orchestrion `orchestrion:ai:*` channels */ export const vercelAiChannelIntegration = defineIntegration(_vercelAiChannelIntegration); diff --git a/packages/server-utils/src/orchestrion/config/vercel-ai.ts b/packages/server-utils/src/orchestrion/config/vercel-ai.ts index bf1be42796f8..708f0f5e71ae 100644 --- a/packages/server-utils/src/orchestrion/config/vercel-ai.ts +++ b/packages/server-utils/src/orchestrion/config/vercel-ai.ts @@ -9,18 +9,30 @@ export const vercelAiConfig = [ // `streamText` returns its result synchronously (streaming is lazy), so it's // `Sync`; the subscriber binds the span via `bindTracingChannelToSpan`, which // ends it when the (synchronous) call returns. - ...vercelAiV6Entries('generateText', 'generateText', 'Async'), - ...vercelAiV6Entries('streamText', 'streamText', 'Sync'), - ...vercelAiV6Entries('embed', 'embed', 'Async'), - ...vercelAiV6Entries('embedMany', 'embedMany', 'Async'), - ...vercelAiV6Entries('executeToolCall', 'executeToolCall', 'Async'), - ...vercelAiV6Entries('resolveLanguageModel', 'resolveLanguageModel', 'Sync'), + ...vercelAiEntries('>=6.0.0 <7.0.0', 'generateText', 'generateText', 'Async'), + ...vercelAiEntries('>=6.0.0 <7.0.0', 'streamText', 'streamText', 'Sync'), + ...vercelAiEntries('>=6.0.0 <7.0.0', 'embed', 'embed', 'Async'), + ...vercelAiEntries('>=6.0.0 <7.0.0', 'embedMany', 'embedMany', 'Async'), + ...vercelAiEntries('>=6.0.0 <7.0.0', 'executeToolCall', 'executeToolCall', 'Async'), + ...vercelAiEntries('>=6.0.0 <7.0.0', 'resolveLanguageModel', 'resolveLanguageModel', 'Sync'), + // Vercel AI v5: same top-level entry points as v6 and the same + // `resolveLanguageModel` chokepoint, so it reuses the same channels and + // subscriber. v5 has no per-call `executeToolCall` export (only the batch + // `executeTools`) and no `rerank`, so those are omitted — the subscriber + // spans v5 tool calls by patching each tool's `execute` instead. + ...vercelAiEntries('>=5.0.0 <6.0.0', 'generateText', 'generateText', 'Async'), + ...vercelAiEntries('>=5.0.0 <6.0.0', 'streamText', 'streamText', 'Sync'), + ...vercelAiEntries('>=5.0.0 <6.0.0', 'embed', 'embed', 'Async'), + ...vercelAiEntries('>=5.0.0 <6.0.0', 'embedMany', 'embedMany', 'Async'), + ...vercelAiEntries('>=5.0.0 <6.0.0', 'resolveLanguageModel', 'resolveLanguageModel', 'Sync'), ] satisfies InstrumentationConfig[]; export const vercelAiChannels = { - // Vercel AI (`ai`) v6: orchestrion injects these so the same channel-based + // Vercel AI (`ai`) v5 & v6: orchestrion injects these so the same channel-based // integration that consumes `ai`'s native `ai:telemetry` channel (v7) can - // also instrument v6. Each maps to a top-level function in `ai`'s bundle. + // also instrument v5/v6. Each maps to a top-level function in `ai`'s bundle. + // v5 and v6 share the same channel names (the subscriber is version-agnostic); + // `VERCEL_AI_EXECUTE_TOOL_CALL` is v6-only (v5 has no `executeToolCall` export). VERCEL_AI_GENERATE_TEXT: 'orchestrion:ai:generateText', VERCEL_AI_STREAM_TEXT: 'orchestrion:ai:streamText', VERCEL_AI_EMBED: 'orchestrion:ai:embed', @@ -49,10 +61,15 @@ export const vercelAiChannels = { * function needs one config entry per file (the app loads whichever matches its * module system). This expands a single target into both. */ -function vercelAiV6Entries(channelName: string, functionName: string, kind: 'Async' | 'Sync'): InstrumentationConfig[] { +function vercelAiEntries( + versionRange: string, + channelName: string, + functionName: string, + kind: 'Async' | 'Sync', +): InstrumentationConfig[] { return ['dist/index.js', 'dist/index.mjs'].map(filePath => ({ channelName, - module: { name: 'ai', versionRange: '>=6.0.0 <7.0.0', filePath }, + module: { name: 'ai', versionRange, filePath }, functionQuery: { functionName, kind }, })); } diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index 1f25d48edff4..da71fd98ea4e 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -530,7 +530,7 @@ function partsFromTextAndToolCalls(text: unknown, toolCalls: unknown): Array= 7 publishes a normalized `ai:telemetry` tracing channel natively - * (consumed by `subscribeVercelAiTracingChannel`). v6 has no such channel, so + * (consumed by `subscribeVercelAiTracingChannel`). v5/v6 have no such channel, so * orchestrion injects `orchestrion:ai:*` channels around the top-level * functions (see `orchestrion/config/index.ts`). The injected channels carry only the * wrapped call's `{ arguments, result, error }` — NOT v7's normalized `event` * object — so this adapter reconstructs an equivalent {@link VercelAiChannelMessage} - * from v6's argument/result shapes and delegates to the SAME span-building core + * from v5/v6's argument/result shapes and delegates to the SAME span-building core * (`createSpanFromMessage` / `enrichSpanOnEnd`) the v7 subscriber uses, so the - * emitted spans are identical between v6 and v7. + * emitted spans are identical between v5, v6 and v7. * * The model call (`languageModelCall` / `generate_content` span) has no * injectable definition in `ai`, so we instead wrap `resolveLanguageModel` (the - * single chokepoint every model call flows through) and monkey-patch - * `doGenerate`/`doStream` on the returned model. + * single chokepoint every model call flows through, present in both v5 and v6) + * and monkey-patch `doGenerate`/`doStream` on the returned model. + * + * Tool-call spans differ by version: v6 exposes a per-call `executeToolCall` + * function orchestrion wraps into its own channel. v5 has no such export (only a + * batch `executeTools`), so instead we monkey-patch each tool's `execute` from + * the operation's `tools` — but only when the `executeToolCall` channel is never + * seen (i.e. we're on v5), so v6 never double-counts tool spans. */ /** Shape orchestrion's transform attaches to the tracing-channel context. */ @@ -51,10 +59,20 @@ interface ResolvedModel { } const PATCHED = Symbol('SentryVercelAiModelPatched'); +const TOOL_PATCHED = Symbol('SentryVercelAiToolPatched'); /** A resolved model with our patch bookkeeping (idempotency flag). */ type PatchableModel = ResolvedModel & { [PATCHED]?: boolean }; +/** A tool definition off an operation's `tools`, with our patch bookkeeping. */ +type PatchableTool = { execute?: (...args: unknown[]) => unknown; [TOOL_PATCHED]?: boolean }; + +// Set the first time the v6-only `executeToolCall` channel fires. It gates the v5 tool-`execute` +// patching: on v6 every tool call flows through `executeToolCall` (its own span-emitting channel), so +// the channel fires before any tool `execute` runs and our patched `execute` becomes a no-op — no +// duplicate tool spans. On v5 the channel is never published, so the patch is what emits tool spans. +let sawExecuteToolCallChannel = false; + // Per-operation correlation id. No Date/random (unavailable / non-deterministic) — a counter is enough. let callIdCounter = 0; function nextCallId(): string { @@ -140,17 +158,21 @@ export function subscribeVercelAiOrchestrionChannels( bindOperation( tracingChannel, CHANNELS.VERCEL_AI_EXECUTE_TOOL_CALL, - (callOptions, telemetry) => ({ - type: 'executeTool', - // v6 carries the tool definitions on the executeToolCall args (a record keyed by name); - // the shared core reads the matching tool's `description` for the span. - event: { - callId: nextCallId(), - toolCall: callOptions.toolCall, - tools: callOptions.tools, - ...recording(telemetry), - }, - }), + (callOptions, telemetry) => { + // Only v6 publishes this channel; seeing it disables the v5 tool-`execute` patch (see flag docs). + sawExecuteToolCallChannel = true; + return { + type: 'executeTool', + // v6 carries the tool definitions on the executeToolCall args (a record keyed by name); + // the shared core reads the matching tool's `description` for the span. + event: { + callId: nextCallId(), + toolCall: callOptions.toolCall, + tools: callOptions.tools, + ...recording(telemetry), + }, + }; + }, options, ); subscribeResolveLanguageModel(tracingChannel, CHANNELS.VERCEL_AI_RESOLVE_LANGUAGE_MODEL, options); @@ -203,6 +225,11 @@ function bindOperation( callIdBySpan.set(span, callId); } recordingBySpan.set(span, recording(telemetry)); + // v5 has no `executeToolCall` channel, so patch each tool's `execute` to emit the tool-call span. + // A no-op on v6 (guarded inside `patchToolExecute` once the `executeToolCall` channel is seen). + if (isRecord(callOptions.tools)) { + patchOperationTools(callOptions.tools, options); + } } return span; }; @@ -394,6 +421,90 @@ function patchModelMethod( }; } +/** + * Patch every executable tool on an operation's `tools` (a record keyed by tool name) so each + * invocation emits a `gen_ai.execute_tool` span. v5 routes tool execution through the batch + * `executeTools`, which binds `tool.execute` at call time — so replacing the `execute` property here + * (before the call runs) is picked up. Tools without an `execute` (client-side tools) are skipped, + * matching `executeTools`. + */ +function patchOperationTools(tools: Record, options: VercelAiChannelOptions): void { + // This runs inside the channel `start` transform (no upstream try/catch), so a throw here would break + // the user's `ai` call. Instrumentation must never do that — degrade to no tool span instead. + try { + for (const [toolName, tool] of Object.entries(tools)) { + if (isRecord(tool)) { + patchToolExecute(toolName, tool as PatchableTool, tools, options); + } + } + } catch { + DEBUG_BUILD && debug.log('Vercel AI orchestrion tool patching failed.'); + } +} + +function patchToolExecute( + toolName: string, + tool: PatchableTool, + tools: Record, + options: VercelAiChannelOptions, +): void { + const original = tool.execute; + if (typeof original !== 'function' || tool[TOOL_PATCHED]) { + return; + } + tool[TOOL_PATCHED] = true; + tool.execute = function (this: unknown, input: unknown, ...rest: unknown[]): unknown { + // On v6 the `executeToolCall` channel already spanned this call before `execute` ran; don't + // double-count. Also skip if there's no enclosing operation span (telemetry disabled for the call). + const parent = resolveModelCallParent(); + if (sawExecuteToolCallChannel || !parent) { + return original.apply(this, [input, ...rest]); + } + + // v5 passes `{ toolCallId, messages, abortSignal, ... }` as the second argument to `execute`. + const callOptions = isRecord(rest[0]) ? rest[0] : {}; + const message: VercelAiChannelMessage = { + type: 'executeTool', + event: { + callId: callIdBySpan.get(parent), + toolCall: { toolName, toolCallId: asString(callOptions.toolCallId), input }, + // The `tools` record (keyed by name) lets the shared core backfill the tool's `description`. + tools, + // Inherit the enclosing operation's per-call recording flags so tool inputs/outputs are recorded + // whenever they are on the parent `invoke_agent` span. + ...recordingBySpan.get(parent), + }, + }; + const span = withActiveSpan(parent, () => createSpanFromMessage(message, options)); + // `executeTool` always opens a span; the guard just keeps the wrapper safe if that changes. + if (!span) { + return original.apply(this, [input, ...rest]); + } + + // v5's `executeTools` catches a thrown tool error and turns it into `tool-error` content rather + // than rejecting, so the user never sees a rejection — we must capture it here. Rethrow so + // `executeTools` still produces its `tool-error` result and the operation continues normally. + const failSpan = (error: unknown): never => { + captureToolError(span, message, error); + span.end(); + throw error; + }; + + try { + const result = Promise.resolve(original.apply(this, [input, ...rest])); + return result.then(value => { + // The shared core (matching v6/v7) expects the tool result nested under `output`. + message.result = { output: value }; + enrichSpanOnEnd(span, message, options); + span.end(); + return value; + }, failSpan); + } catch (error) { + return failSpan(error); + } + }; +} + function buildTextMessage(type: 'generateText' | 'streamText'): MessageBuilder { return (options, telemetry) => ({ type, @@ -422,7 +533,15 @@ function normalizePromptMessages(options: Record): unknown { } function recording(telemetry: Record): { recordInputs: unknown; recordOutputs: unknown } { - return { recordInputs: telemetry.recordInputs, recordOutputs: telemetry.recordOutputs }; + // Match the OTel integration's per-call default: an explicit `recordInputs`/`recordOutputs` wins, but a + // call that merely enables telemetry (`isEnabled: true`, no explicit flags) defaults both to `true`. + // Unlike v7's native `ai:telemetry` channel, the orchestrion channels expose `isEnabled`, so — as noted + // in `resolveRecording` — we CAN reproduce that default here (the native-channel subscriber cannot). + const enabledDefault = telemetry.isEnabled === true ? true : undefined; + return { + recordInputs: telemetry.recordInputs ?? enabledDefault, + recordOutputs: telemetry.recordOutputs ?? enabledDefault, + }; } function modelFields(model: unknown): { provider?: string; modelId?: string } { From 4e43abc0b95a47b5a4c7f2f13886cfacde58ed10 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Mon, 6 Jul 2026 13:31:29 +0200 Subject: [PATCH 2/5] fix test --- .../suites/tracing/vercelai/v5/test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index ed0c8b73114a..3fc8edf15566 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -20,9 +20,6 @@ import { import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { isOrchestrionEnabled } from '../../../../utils'; -// In orchestrion mode the channel-based integration emits the spans (origin `auto.vercelai.channel`); -// otherwise the OTel-based integration does (`auto.vercelai.otel`). The spans are otherwise identical, -// so the same assertions run in both modes — only the origin differs. const expectedOrigin = isOrchestrionEnabled() ? 'auto.vercelai.channel' : 'auto.vercelai.otel'; describe('Vercel AI integration (v5)', () => { @@ -274,6 +271,9 @@ describe('Vercel AI integration (v5)', () => { let errorEvent: Event | undefined; await createRunner() + // The tool error is captured while the tool is running (mid-transaction), so the error event + // and the transaction/span envelopes can arrive in either order — assert content, not wire order. + .unordered() .expect({ transaction: transaction => { transactionEvent = transaction; From 0c9005843a58ec36f889eb1a868d87372732f3f8 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Tue, 7 Jul 2026 12:11:29 +0200 Subject: [PATCH 3/5] ref --- .../vercel-ai-orchestrion-subscriber.ts | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts index db059e8d4f55..5227b1dc5072 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -36,8 +36,11 @@ import { * Tool-call spans differ by version: v6 exposes a per-call `executeToolCall` * function orchestrion wraps into its own channel. v5 has no such export (only a * batch `executeTools`), so instead we monkey-patch each tool's `execute` from - * the operation's `tools` — but only when the `executeToolCall` channel is never - * seen (i.e. we're on v5), so v6 never double-counts tool spans. + * the operation's `tools`. On v6 that patch is inert: `executeToolCall` runs + * `execute` inside its own tool-call span (the active async context), so the + * patch sees that span as its parent and skips — only v5 (where the parent is + * the enclosing `invoke_agent` span) emits the patched span, so v6 never + * double-counts tool spans. */ /** Shape orchestrion's transform attaches to the tracing-channel context. */ @@ -67,12 +70,6 @@ type PatchableModel = ResolvedModel & { [PATCHED]?: boolean }; /** A tool definition off an operation's `tools`, with our patch bookkeeping. */ type PatchableTool = { execute?: (...args: unknown[]) => unknown; [TOOL_PATCHED]?: boolean }; -// Set the first time the v6-only `executeToolCall` channel fires. It gates the v5 tool-`execute` -// patching: on v6 every tool call flows through `executeToolCall` (its own span-emitting channel), so -// the channel fires before any tool `execute` runs and our patched `execute` becomes a no-op — no -// duplicate tool spans. On v5 the channel is never published, so the patch is what emits tool spans. -let sawExecuteToolCallChannel = false; - // Per-operation correlation id. No Date/random (unavailable / non-deterministic) — a counter is enough. let callIdCounter = 0; function nextCallId(): string { @@ -86,6 +83,10 @@ const messages = new WeakMap(); // parent against this set (so it never mis-attributes to the enclosing `main`/user span) and reads the // parent's `callId` so its span can be named after the operation (e.g. `ai.streamText.doStream`). const operationSpans = new WeakSet(); +// Tool-call spans — only v6's `executeToolCall` channel opens these. The v5 tool-`execute` patch +// consults this to avoid double-counting: `executeToolCall` runs `execute` inside its span (which is +// therefore the active parent), so a resolved parent in this set means v6 already spanned the call. +const toolCallSpans = new WeakSet(); const callIdBySpan = new WeakMap(); // The operation's per-call recording flags (`experimental_telemetry.recordInputs/recordOutputs`), keyed by // its span. A model call carries no telemetry of its own, so it inherits the enclosing operation's flags — @@ -158,21 +159,17 @@ export function subscribeVercelAiOrchestrionChannels( bindOperation( tracingChannel, CHANNELS.VERCEL_AI_EXECUTE_TOOL_CALL, - (callOptions, telemetry) => { - // Only v6 publishes this channel; seeing it disables the v5 tool-`execute` patch (see flag docs). - sawExecuteToolCallChannel = true; - return { - type: 'executeTool', - // v6 carries the tool definitions on the executeToolCall args (a record keyed by name); - // the shared core reads the matching tool's `description` for the span. - event: { - callId: nextCallId(), - toolCall: callOptions.toolCall, - tools: callOptions.tools, - ...recording(telemetry), - }, - }; - }, + (callOptions, telemetry) => ({ + type: 'executeTool', + // v6 carries the tool definitions on the executeToolCall args (a record keyed by name); + // the shared core reads the matching tool's `description` for the span. + event: { + callId: nextCallId(), + toolCall: callOptions.toolCall, + tools: callOptions.tools, + ...recording(telemetry), + }, + }), options, ); subscribeResolveLanguageModel(tracingChannel, CHANNELS.VERCEL_AI_RESOLVE_LANGUAGE_MODEL, options); @@ -220,13 +217,18 @@ function bindOperation( if (span) { messages.set(data, message); operationSpans.add(span); + // v6's `executeToolCall` span: tracked so the v5 tool-`execute` patch can recognize (and skip) + // tool calls it runs, since that patch sees this span as its active parent (see `patchToolExecute`). + if (message.type === 'executeTool') { + toolCallSpans.add(span); + } const callId = asString(message.event.callId); if (callId) { callIdBySpan.set(span, callId); } recordingBySpan.set(span, recording(telemetry)); // v5 has no `executeToolCall` channel, so patch each tool's `execute` to emit the tool-call span. - // A no-op on v6 (guarded inside `patchToolExecute` once the `executeToolCall` channel is seen). + // Inert on v6 (guarded inside `patchToolExecute` when the parent is `executeToolCall`'s own span). if (isRecord(callOptions.tools)) { patchOperationTools(callOptions.tools, options); } @@ -454,10 +456,12 @@ function patchToolExecute( } tool[TOOL_PATCHED] = true; tool.execute = function (this: unknown, input: unknown, ...rest: unknown[]): unknown { - // On v6 the `executeToolCall` channel already spanned this call before `execute` ran; don't - // double-count. Also skip if there's no enclosing operation span (telemetry disabled for the call). + // Skip if there's no enclosing operation span (telemetry disabled for the call), or if that parent + // is itself a tool-call span — the latter means v6's `executeToolCall` already opened a span for + // this call and is running `execute` inside it, so spanning again here would double-count. On v5 + // the parent is the enclosing `invoke_agent` operation span, so we proceed. const parent = resolveModelCallParent(); - if (sawExecuteToolCallChannel || !parent) { + if (!parent || toolCallSpans.has(parent)) { return original.apply(this, [input, ...rest]); } From cc0ee3a851edea2f0d841bb8ef160f8c03208c68 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 8 Jul 2026 09:07:09 +0200 Subject: [PATCH 4/5] streamline vercel AI things --- .../src/orchestrion/config/vercel-ai.ts | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/server-utils/src/orchestrion/config/vercel-ai.ts b/packages/server-utils/src/orchestrion/config/vercel-ai.ts index 708f0f5e71ae..d2be04f4984f 100644 --- a/packages/server-utils/src/orchestrion/config/vercel-ai.ts +++ b/packages/server-utils/src/orchestrion/config/vercel-ai.ts @@ -9,22 +9,14 @@ export const vercelAiConfig = [ // `streamText` returns its result synchronously (streaming is lazy), so it's // `Sync`; the subscriber binds the span via `bindTracingChannelToSpan`, which // ends it when the (synchronous) call returns. - ...vercelAiEntries('>=6.0.0 <7.0.0', 'generateText', 'generateText', 'Async'), - ...vercelAiEntries('>=6.0.0 <7.0.0', 'streamText', 'streamText', 'Sync'), - ...vercelAiEntries('>=6.0.0 <7.0.0', 'embed', 'embed', 'Async'), - ...vercelAiEntries('>=6.0.0 <7.0.0', 'embedMany', 'embedMany', 'Async'), + // Vercel AI v5: same top-level entry points as v6 + ...vercelAiEntries('>=5.0.0 <7.0.0', 'generateText', 'generateText', 'Async'), + ...vercelAiEntries('>=5.0.0 <7.0.0', 'streamText', 'streamText', 'Sync'), + ...vercelAiEntries('>=5.0.0 <7.0.0', 'embed', 'embed', 'Async'), + ...vercelAiEntries('>=5.0.0 <7.0.0', 'embedMany', 'embedMany', 'Async'), + ...vercelAiEntries('>=5.0.0 <7.0.0', 'resolveLanguageModel', 'resolveLanguageModel', 'Sync'), + // This only exists in >= v6 ...vercelAiEntries('>=6.0.0 <7.0.0', 'executeToolCall', 'executeToolCall', 'Async'), - ...vercelAiEntries('>=6.0.0 <7.0.0', 'resolveLanguageModel', 'resolveLanguageModel', 'Sync'), - // Vercel AI v5: same top-level entry points as v6 and the same - // `resolveLanguageModel` chokepoint, so it reuses the same channels and - // subscriber. v5 has no per-call `executeToolCall` export (only the batch - // `executeTools`) and no `rerank`, so those are omitted — the subscriber - // spans v5 tool calls by patching each tool's `execute` instead. - ...vercelAiEntries('>=5.0.0 <6.0.0', 'generateText', 'generateText', 'Async'), - ...vercelAiEntries('>=5.0.0 <6.0.0', 'streamText', 'streamText', 'Sync'), - ...vercelAiEntries('>=5.0.0 <6.0.0', 'embed', 'embed', 'Async'), - ...vercelAiEntries('>=5.0.0 <6.0.0', 'embedMany', 'embedMany', 'Async'), - ...vercelAiEntries('>=5.0.0 <6.0.0', 'resolveLanguageModel', 'resolveLanguageModel', 'Sync'), ] satisfies InstrumentationConfig[]; export const vercelAiChannels = { From a8f82ceec1c60438e183f5ddafd82803ea139838 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 8 Jul 2026 09:31:57 +0200 Subject: [PATCH 5/5] better revalidate --- .../utils/runner/createEsmAndCjsTests.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/dev-packages/node-integration-tests/utils/runner/createEsmAndCjsTests.ts b/dev-packages/node-integration-tests/utils/runner/createEsmAndCjsTests.ts index 5def48851504..91de78c7c5ab 100644 --- a/dev-packages/node-integration-tests/utils/runner/createEsmAndCjsTests.ts +++ b/dev-packages/node-integration-tests/utils/runner/createEsmAndCjsTests.ts @@ -366,10 +366,15 @@ const NPM_INSTALL_RETRY_DELAY_MS = 2_000; async function npmInstallWithRetry(cwd: string, deps: string[]): Promise { for (let attempt = 1; attempt <= NPM_INSTALL_MAX_RETRIES; attempt++) { try { - const { stdout, stderr } = await execPromise('npm install --prefer-offline --silent --no-audit --no-fund', { - cwd, - encoding: 'utf8', - }); + const { stdout, stderr } = await execPromise( + // We only use --prefer-offline on the first attempt to try to read from cache + // in follow ups we just try to install in any way + `npm install ${attempt === 1 ? '--prefer-offline' : ''} --no-audit --no-fund`, + { + cwd, + encoding: 'utf8', + }, + ); if (process.env.DEBUG) { // eslint-disable-next-line no-console @@ -388,8 +393,10 @@ async function npmInstallWithRetry(cwd: string, deps: string[]): Promise { ); await new Promise(resolve => setTimeout(resolve, NPM_INSTALL_RETRY_DELAY_MS)); } else { + // oxlint-disable-next-line no-console + console.error(error); throw new Error( - `Failed to install additionalDependencies in tmp dir ${cwd} after ${NPM_INSTALL_MAX_RETRIES} attempts: ${error}`, + `Failed to install additionalDependencies in tmp dir ${cwd} after ${NPM_INSTALL_MAX_RETRIES} attempts`, ); } }