From 1a007c442f6063de0d4d074f91cfc88fde2d15f5 Mon Sep 17 00:00:00 2001 From: Dominik Buszowiecki Date: Thu, 22 Oct 2026 16:36:20 -0400 Subject: [PATCH 1/5] feat(search-events): Track assisted query outcome for Seer translations When Seer's search agent translates a natural language query, report whether the executed query returned results via the `assisted_query.outcome` log and distribution metric, matching `trackAiQueryOutcome` in Sentry's frontend. The referrer is `assisted_query.mcp.` (spans reported as traces), so the assisted query dashboard can count MCP alongside the search bar. Outcomes are `has_results`, `empty_results`, and `error_on_load`. Translation-time failures are already reported by Seer, and queries translated by the embedded agent are not tracked. Co-Authored-By: Claude Opus 5.5 --- .../src/tools/catalog/search-events.test.ts | 176 ++++++++++++++++++ .../src/tools/support/search-events/search.ts | 82 +++++--- .../src/tools/support/search-events/seer.ts | 111 ++++++++++- 3 files changed, 340 insertions(+), 29 deletions(-) diff --git a/packages/mcp-core/src/tools/catalog/search-events.test.ts b/packages/mcp-core/src/tools/catalog/search-events.test.ts index 33766f851..a52781898 100644 --- a/packages/mcp-core/src/tools/catalog/search-events.test.ts +++ b/packages/mcp-core/src/tools/catalog/search-events.test.ts @@ -1,3 +1,4 @@ +import { metrics } from "@sentry/core"; import { mswServer } from "@sentry/mcp-server-mocks"; import { APICallError, generateText } from "ai"; import { HttpResponse, http } from "msw"; @@ -15,6 +16,14 @@ vi.mock("@ai-sdk/openai", () => { }; }); +vi.mock("@sentry/core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + metrics: { ...actual.metrics, distribution: vi.fn() }, + }; +}); + vi.mock("ai", async (importOriginal) => { const actual = await importOriginal(); return { @@ -3758,8 +3767,11 @@ describe("search_events", () => { () => HttpResponse.json({ session, sentry_run_id: "run-uuid" }), ); + const mockDistribution = vi.mocked(metrics.distribution); + beforeEach(() => { mockSeerStart.mockClear(); + mockDistribution.mockClear(); mswServer.use( mockProject, http.post( @@ -4399,5 +4411,169 @@ describe("search_events", () => { expect(mockGenerateText).toHaveBeenCalled(); }); + + describe("assisted query outcome", () => { + const mockSeerEvents = ( + resolver: Parameters[1], + ): Parameters => [ + mockOrganization([ + "mcp-search-events-seer-translate", + "seer-assisted-query-cross-event-explorer", + ]), + mockSeerState({ + status: "completed", + final_response: { responses: [seerQuery], unsupported_reason: null }, + }), + http.get( + "https://sentry.io/api/0/organizations/test-org/events/", + resolver, + ), + ]; + + it("should report has_results when the Seer query returns rows", async () => { + mswServer.use( + ...mockSeerEvents(() => + HttpResponse.json({ + data: [ + { "span.description": "GET /a", "p95(span.duration)": 1 }, + { "span.description": "GET /b", "p95(span.duration)": 2 }, + ], + }), + ), + ); + + await searchEvents.handler(seerParams, context); + + expect(mockDistribution).toHaveBeenCalledTimes(1); + expect(mockDistribution).toHaveBeenCalledWith( + "assisted_query.outcome", + 2, + { + attributes: { + dataset: "spans", + mode: "aggregate", + org_slug: "test-org", + referrer: "assisted_query.mcp.traces", + run_id: "run-uuid", + conversation_url: + "https://sentry.io/organizations/sentry/explore/agents/conversations/run-uuid/", + outcome: "has_results", + code_mode: false, + cross_event_enabled: true, + }, + }, + ); + }); + + it("should report empty_results when the Seer query returns no rows", async () => { + mswServer.use(...mockSeerEvents(() => HttpResponse.json({ data: [] }))); + + await searchEvents.handler(seerParams, context); + + expect(mockDistribution).toHaveBeenCalledWith( + "assisted_query.outcome", + 0, + { + attributes: expect.objectContaining({ outcome: "empty_results" }), + }, + ); + }); + + it("should report error_on_load when the Seer query fails", async () => { + mswServer.use( + ...mockSeerEvents(() => + HttpResponse.json({ detail: "Invalid query" }, { status: 400 }), + ), + ); + + await expect( + searchEvents.handler(seerParams, context), + ).rejects.toThrow(); + + expect(mockDistribution).toHaveBeenCalledWith( + "assisted_query.outcome", + 0, + { + attributes: expect.objectContaining({ + outcome: "error_on_load", + error_reason: expect.stringContaining("Invalid query"), + }), + }, + ); + }); + + it("should count non-empty buckets for a Seer time series", async () => { + mswServer.use( + mockOrganization(["mcp-search-events-seer-translate"]), + mockSeerState({ + status: "completed", + final_response: { + responses: [ + { + ...seerQuery, + query: "", + group_by: [], + visualization: [ + { chart_type: 1, y_axes: ["count()"], interval: "1d" }, + ], + sort: "-count()", + stats_period: "7d", + }, + ], + unsupported_reason: null, + }, + }), + http.get( + "https://sentry.io/api/0/organizations/test-org/events-timeseries/", + () => + HttpResponse.json({ + timeSeries: [ + { + yAxis: "count()", + values: [ + { timestamp: 1757548800000, value: 0, incomplete: false }, + { timestamp: 1757635200000, value: 8, incomplete: false }, + { + timestamp: 1757721600000, + value: null, + incomplete: false, + }, + ], + meta: { interval: 86400000 }, + }, + ], + }), + ), + ); + + await searchEvents.handler(seerParams, context); + + expect(mockDistribution).toHaveBeenCalledWith( + "assisted_query.outcome", + 1, + { + attributes: expect.objectContaining({ outcome: "has_results" }), + }, + ); + }); + + it("should not report when the embedded agent translates the query", async () => { + mockGenerateText.mockResolvedValueOnce( + mockAIResponse("spans", "span.op:http.client"), + ); + mswServer.use( + mockOrganization([]), + http.get( + "https://sentry.io/api/0/organizations/test-org/events/", + () => HttpResponse.json({ data: [] }), + ), + ); + + await searchEvents.handler(seerParams, context); + + expect(mockGenerateText).toHaveBeenCalled(); + expect(mockDistribution).not.toHaveBeenCalled(); + }); + }); }); }); diff --git a/packages/mcp-core/src/tools/support/search-events/search.ts b/packages/mcp-core/src/tools/support/search-events/search.ts index 49931d663..b7f6cca82 100644 --- a/packages/mcp-core/src/tools/support/search-events/search.ts +++ b/packages/mcp-core/src/tools/support/search-events/search.ts @@ -46,7 +46,11 @@ import { formatReplayResults, isValidReplaySort, } from "./replays"; -import { isSeerSearchDataset, translateWithSeer } from "./seer"; +import { + isSeerSearchDataset, + trackAssistedQueryOutcome, + translateWithSeer, +} from "./seer"; import { formatEventsValidationResults, isAggregateQuery, @@ -870,6 +874,19 @@ export async function runSearchEvents( const withLeadingNote = (text: string): string => leadingNote ? `${leadingNote}\n\n${text}` : text; + // Only Seer's translations are tracked, not the embedded agent's. + const trackSeerOutcome = (resultCount: number, error?: unknown): void => { + if (seerTranslation && isSeerSearchDataset(params.dataset)) { + trackAssistedQueryOutcome({ + dataset: params.dataset, + organizationSlug, + translation: seerTranslation, + resultCount, + error, + }); + } + }; + if (dataset === "replays") { const replaySort = sortParam || DEFAULT_REPLAY_SORT; if (!isValidReplaySort(replaySort)) { @@ -950,15 +967,27 @@ export async function runSearchEvents( // request shape — fields + orderby — which is not what a timeseries // sends (yAxis + interval, no fields/sort). events-timeseries validates // the query server-side, so a bad query still surfaces as an API error. - const series = await apiService.getEventsTimeSeries({ - organizationSlug, - query: timeSeriesQuery, - yAxis: timeSeries.yAxis, - interval: timeSeries.interval ?? undefined, - projectId, - dataset, - ...timeParams, - }); + let series: Awaited>; + try { + series = await apiService.getEventsTimeSeries({ + organizationSlug, + query: timeSeriesQuery, + yAxis: timeSeries.yAxis, + interval: timeSeries.interval ?? undefined, + projectId, + dataset, + ...timeParams, + }); + } catch (error) { + trackSeerOutcome(0, error); + throw error; + } + // A series always has buckets, so count the ones with data. + trackSeerOutcome( + series.timeSeries + .flatMap((s) => s.values) + .filter((point) => point.value != null && point.value !== 0).length, + ); const statsUrl = apiService.getEventsExplorerUrl( organizationSlug, timeSeriesQuery, @@ -1025,11 +1054,13 @@ export async function runSearchEvents( if (!lastValidation.valid) { const formatted = formatEventsValidationResults(lastValidation); - throw new UserInputError( + const error = new UserInputError( formatted ? `Search validation failed:\n${formatted}` : "Search validation failed.", ); + trackSeerOutcome(0, error); + throw error; } const finalRequestFields = buildRequestFields(dataset, fields); @@ -1039,17 +1070,23 @@ export async function runSearchEvents( environment, ); - const eventsResponse = await apiService.searchEvents({ - organizationSlug, - query: sentryQuery, - fields: finalRequestFields, - limit: params.limit, - projectId, - dataset, - sort: sortParam, - crossEventQueries: seerTranslation?.crossEventQueries, - ...timeParams, - }); + let eventsResponse: Awaited>; + try { + eventsResponse = await apiService.searchEvents({ + organizationSlug, + query: sentryQuery, + fields: finalRequestFields, + limit: params.limit, + projectId, + dataset, + sort: sortParam, + crossEventQueries: seerTranslation?.crossEventQueries, + ...timeParams, + }); + } catch (error) { + trackSeerOutcome(0, error); + throw error; + } const aggregateFunctions = fields.filter( (field) => field.includes("(") && field.includes(")"), @@ -1084,6 +1121,7 @@ export async function runSearchEvents( "gen_ai.tool.call.result.count", eventData.length, ); + trackSeerOutcome(eventData.length); const conversationId = extractConversationIdFromSearchQuery(sentryQuery); const explorerUrl = conversationId diff --git a/packages/mcp-core/src/tools/support/search-events/seer.ts b/packages/mcp-core/src/tools/support/search-events/seer.ts index b146a6742..7e8903422 100644 --- a/packages/mcp-core/src/tools/support/search-events/seer.ts +++ b/packages/mcp-core/src/tools/support/search-events/seer.ts @@ -1,10 +1,11 @@ +import { metrics } from "@sentry/core"; import type { z } from "zod"; import type { CrossEventQueries, SentryApiService } from "../../../api-client"; import { ApiAuthenticationError, type SearchAgentQuerySchema, } from "../../../api-client/index"; -import { logWarn } from "../../../telem/logging"; +import { logInfo, logWarn } from "../../../telem/logging"; import { normalizeEventsDataset, type PublicEventsDataset, @@ -48,28 +49,114 @@ export interface SeerSearchTranslation { explanation: string; // Correctness warnings must be shown even when explanations are disabled. warning?: string; + // Identifies the run in the assisted query outcome metric. + runId: string; + mode: AssistedQueryMode; + organizationFeatures: string[]; } -async function hasSeerSearchAgentAccess( +// Seer doesn't return results, so the caller reports whether the translated +// query found anything. Mirrors `trackAiQueryOutcome` in Sentry's frontend, +// which reports the same metric with an `assisted_query.search_bar.*` referrer. +// See `static/app/components/searchQueryBuilder/askSeerCombobox/utils.ts`. +const ASSISTED_QUERY_OUTCOME = "assisted_query.outcome"; + +// The frontend's dataset and referrer names for each Seer dataset. +const ASSISTED_QUERY_DATASETS = { + errors: "errors", + logs: "logs", + spans: "spans", + metrics: "tracemetrics", +} as const satisfies Record; + +const ASSISTED_QUERY_REFERRERS = { + errors: "assisted_query.mcp.errors", + logs: "assisted_query.mcp.logs", + spans: "assisted_query.mcp.traces", + metrics: "assisted_query.mcp.tracemetrics", +} as const satisfies Record; + +export type AssistedQueryMode = "samples" | "aggregate"; + +export type AssistedQueryOutcome = + | "has_results" + | "empty_results" + | "error_on_load"; + +/** + * Reports whether a Seer-translated query returned results once executed. + * Pass `error` when the query failed to run. + */ +export function trackAssistedQueryOutcome({ + dataset, + organizationSlug, + translation, + resultCount, + error, +}: { + dataset: SeerSearchDataset; + organizationSlug: string; + translation: SeerSearchTranslation; + resultCount: number; + error?: unknown; +}): void { + const outcome: AssistedQueryOutcome = error + ? "error_on_load" + : resultCount > 0 + ? "has_results" + : "empty_results"; + const errorReason = error + ? error instanceof Error + ? error.message + : String(error) + : undefined; + + const attributes = { + dataset: ASSISTED_QUERY_DATASETS[dataset], + mode: translation.mode, + org_slug: organizationSlug, + referrer: ASSISTED_QUERY_REFERRERS[dataset], + run_id: translation.runId, + // Seer runs are stored as conversations in the `sentry` org. + conversation_url: `https://sentry.io/organizations/sentry/explore/agents/conversations/${encodeURIComponent(translation.runId)}/`, + outcome, + code_mode: translation.organizationFeatures.includes( + "seer-assisted-query-codemode", + ), + cross_event_enabled: translation.organizationFeatures.includes( + "seer-assisted-query-cross-event-explorer", + ), + ...(errorReason ? { error_reason: errorReason } : {}), + }; + + logInfo(ASSISTED_QUERY_OUTCOME, { + extra: { ...attributes, result_count: resultCount }, + }); + metrics.distribution(ASSISTED_QUERY_OUTCOME, resultCount, { attributes }); +} + +// Returns the org's features when Seer translation is enabled, otherwise null. +async function getSeerSearchAgentFeatures( apiService: SentryApiService, organizationSlug: string, -): Promise { +): Promise { // Sentry omits `features` unless explicitly requested. const organization = await apiService.getOrganization(organizationSlug, { includeFeatureFlags: true, detailed: false, }); if (organization.hideAiFeatures) { - return false; + return null; } const features = organization.features ?? []; - return features.includes(SEER_TRANSLATE_FEATURE); + return features.includes(SEER_TRANSLATE_FEATURE) ? features : null; } function toSearchTranslation( result: z.output, dataset: SeerSearchDataset, suggestedProjectIds: number[], + run: { runId: string; organizationFeatures: string[] }, ): SeerSearchTranslation { const aggregates = result.visualization.flatMap((chart) => chart.y_axes); const fields = @@ -158,6 +245,9 @@ function toSearchTranslation( suggestedProjectIds, explanation, warning, + runId: run.runId, + mode: result.mode === "aggregates" ? "aggregate" : "samples", + organizationFeatures: run.organizationFeatures, }; } @@ -182,7 +272,11 @@ export async function translateWithSeer({ query: string; }): Promise { try { - if (!(await hasSeerSearchAgentAccess(apiService, organizationSlug))) { + const organizationFeatures = await getSeerSearchAgentFeatures( + apiService, + organizationSlug, + ); + if (!organizationFeatures) { return null; } @@ -213,7 +307,10 @@ export async function translateWithSeer({ (id) => id !== Number(projectId), ) : []; - return toSearchTranslation(result, dataset, suggestedProjectIds); + return toSearchTranslation(result, dataset, suggestedProjectIds, { + runId: run.sentry_run_id, + organizationFeatures, + }); } if (session?.status === "error") { return null; From 8b4be79d70bd00601b4382a5fd9eedeb7046665e Mon Sep 17 00:00:00 2001 From: Dominik Buszowiecki Date: Fri, 23 Oct 2026 09:46:54 -0400 Subject: [PATCH 2/5] fix(search-events): Track error_on_load for any failure after Seer translation Catch errors once around the whole search instead of per API call, so validation request failures and response shape errors also report error_on_load. Each run reports at most one outcome. Co-Authored-By: Claude Opus 5.5 --- .../src/tools/catalog/search-events.test.ts | 30 +++++ .../src/tools/support/search-events/search.ts | 121 +++++++++++------- 2 files changed, 102 insertions(+), 49 deletions(-) diff --git a/packages/mcp-core/src/tools/catalog/search-events.test.ts b/packages/mcp-core/src/tools/catalog/search-events.test.ts index a52781898..6aece5a9b 100644 --- a/packages/mcp-core/src/tools/catalog/search-events.test.ts +++ b/packages/mcp-core/src/tools/catalog/search-events.test.ts @@ -4502,6 +4502,36 @@ describe("search_events", () => { ); }); + it("should report error_on_load when validating the Seer query fails", async () => { + mswServer.use( + ...mockSeerEvents(() => HttpResponse.json({ data: [] })), + http.get( + "https://sentry.io/api/0/organizations/test-org/events/validate/", + () => + HttpResponse.json( + { detail: "Validation broke" }, + { status: 400 }, + ), + ), + ); + + await expect( + searchEvents.handler(seerParams, context), + ).rejects.toThrow(); + + expect(mockDistribution).toHaveBeenCalledTimes(1); + expect(mockDistribution).toHaveBeenCalledWith( + "assisted_query.outcome", + 0, + { + attributes: expect.objectContaining({ + outcome: "error_on_load", + error_reason: expect.any(String), + }), + }, + ); + }); + it("should count non-empty buckets for a Seer time series", async () => { mswServer.use( mockOrganization(["mcp-search-events-seer-translate"]), diff --git a/packages/mcp-core/src/tools/support/search-events/search.ts b/packages/mcp-core/src/tools/support/search-events/search.ts index b7f6cca82..5e39555b5 100644 --- a/packages/mcp-core/src/tools/support/search-events/search.ts +++ b/packages/mcp-core/src/tools/support/search-events/search.ts @@ -48,6 +48,7 @@ import { } from "./replays"; import { isSeerSearchDataset, + type SeerSearchTranslation, trackAssistedQueryOutcome, translateWithSeer, } from "./seer"; @@ -611,10 +612,56 @@ export function logSearchEventsError( /** * Shared handler for search_events and the dataset-specific search tools. */ +interface SeerOutcomeState { + translation: SeerSearchTranslation | null; + tracked: boolean; +} + +// Only Seer's translations are tracked, not the embedded agent's, and at most +// once per run. +function trackSeerOutcome( + state: SeerOutcomeState, + params: SearchEventsParams, + resultCount: number, + error?: unknown, +): void { + if ( + !state.translation || + state.tracked || + !isSeerSearchDataset(params.dataset) + ) { + return; + } + state.tracked = true; + trackAssistedQueryOutcome({ + dataset: params.dataset, + organizationSlug: params.organizationSlug, + translation: state.translation, + resultCount, + error, + }); +} + export async function runSearchEvents( params: SearchEventsParams, context: ServerContext, options: RunSearchEventsOptions = {}, +): Promise { + // Any failure after Seer translates the query counts as `error_on_load`. + const seerOutcome: SeerOutcomeState = { translation: null, tracked: false }; + try { + return await executeSearchEvents(params, context, options, seerOutcome); + } catch (error) { + trackSeerOutcome(seerOutcome, params, 0, error); + throw error; + } +} + +async function executeSearchEvents( + params: SearchEventsParams, + context: ServerContext, + options: RunSearchEventsOptions, + seerOutcome: SeerOutcomeState, ): Promise { const lockDataset = options.lockDataset === true; const apiService = apiServiceFromContext(context, { @@ -691,6 +738,7 @@ export async function runSearchEvents( query: params.query, }) : null; + seerOutcome.translation = seerTranslation; if (seerTranslation && !projectId) { projectId = "-1"; } @@ -874,19 +922,6 @@ export async function runSearchEvents( const withLeadingNote = (text: string): string => leadingNote ? `${leadingNote}\n\n${text}` : text; - // Only Seer's translations are tracked, not the embedded agent's. - const trackSeerOutcome = (resultCount: number, error?: unknown): void => { - if (seerTranslation && isSeerSearchDataset(params.dataset)) { - trackAssistedQueryOutcome({ - dataset: params.dataset, - organizationSlug, - translation: seerTranslation, - resultCount, - error, - }); - } - }; - if (dataset === "replays") { const replaySort = sortParam || DEFAULT_REPLAY_SORT; if (!isValidReplaySort(replaySort)) { @@ -967,23 +1002,19 @@ export async function runSearchEvents( // request shape — fields + orderby — which is not what a timeseries // sends (yAxis + interval, no fields/sort). events-timeseries validates // the query server-side, so a bad query still surfaces as an API error. - let series: Awaited>; - try { - series = await apiService.getEventsTimeSeries({ - organizationSlug, - query: timeSeriesQuery, - yAxis: timeSeries.yAxis, - interval: timeSeries.interval ?? undefined, - projectId, - dataset, - ...timeParams, - }); - } catch (error) { - trackSeerOutcome(0, error); - throw error; - } + const series = await apiService.getEventsTimeSeries({ + organizationSlug, + query: timeSeriesQuery, + yAxis: timeSeries.yAxis, + interval: timeSeries.interval ?? undefined, + projectId, + dataset, + ...timeParams, + }); // A series always has buckets, so count the ones with data. trackSeerOutcome( + seerOutcome, + params, series.timeSeries .flatMap((s) => s.values) .filter((point) => point.value != null && point.value !== 0).length, @@ -1054,13 +1085,11 @@ export async function runSearchEvents( if (!lastValidation.valid) { const formatted = formatEventsValidationResults(lastValidation); - const error = new UserInputError( + throw new UserInputError( formatted ? `Search validation failed:\n${formatted}` : "Search validation failed.", ); - trackSeerOutcome(0, error); - throw error; } const finalRequestFields = buildRequestFields(dataset, fields); @@ -1070,23 +1099,17 @@ export async function runSearchEvents( environment, ); - let eventsResponse: Awaited>; - try { - eventsResponse = await apiService.searchEvents({ - organizationSlug, - query: sentryQuery, - fields: finalRequestFields, - limit: params.limit, - projectId, - dataset, - sort: sortParam, - crossEventQueries: seerTranslation?.crossEventQueries, - ...timeParams, - }); - } catch (error) { - trackSeerOutcome(0, error); - throw error; - } + const eventsResponse = await apiService.searchEvents({ + organizationSlug, + query: sentryQuery, + fields: finalRequestFields, + limit: params.limit, + projectId, + dataset, + sort: sortParam, + crossEventQueries: seerTranslation?.crossEventQueries, + ...timeParams, + }); const aggregateFunctions = fields.filter( (field) => field.includes("(") && field.includes(")"), @@ -1121,7 +1144,7 @@ export async function runSearchEvents( "gen_ai.tool.call.result.count", eventData.length, ); - trackSeerOutcome(eventData.length); + trackSeerOutcome(seerOutcome, params, eventData.length); const conversationId = extractConversationIdFromSearchQuery(sentryQuery); const explorerUrl = conversationId From edb373885b9e55302d1d0fc90e290f2efe7c48e1 Mon Sep 17 00:00:00 2001 From: Dominik Buszowiecki Date: Fri, 23 Oct 2026 09:49:42 -0400 Subject: [PATCH 3/5] ref(search-events): Rename outcome helper to trackAiQueryOutcome Match the frontend's trackAiQueryOutcome signature and body: dataset, mode, organization, resultCount, runId and error, with the referrer derived from the dataset, error reasons taken from the API detail, and the log sent through Sentry's logger. Co-Authored-By: Claude Opus 5.5 --- .../src/tools/support/search-events/search.ts | 24 ++-- .../src/tools/support/search-events/seer.ts | 119 +++++++++--------- 2 files changed, 77 insertions(+), 66 deletions(-) diff --git a/packages/mcp-core/src/tools/support/search-events/search.ts b/packages/mcp-core/src/tools/support/search-events/search.ts index 5e39555b5..945470021 100644 --- a/packages/mcp-core/src/tools/support/search-events/search.ts +++ b/packages/mcp-core/src/tools/support/search-events/search.ts @@ -47,9 +47,10 @@ import { isValidReplaySort, } from "./replays"; import { + AI_QUERY_DATASETS, isSeerSearchDataset, type SeerSearchTranslation, - trackAssistedQueryOutcome, + trackAiQueryOutcome, translateWithSeer, } from "./seer"; import { @@ -623,7 +624,7 @@ function trackSeerOutcome( state: SeerOutcomeState, params: SearchEventsParams, resultCount: number, - error?: unknown, + error: unknown = false, ): void { if ( !state.translation || @@ -633,12 +634,21 @@ function trackSeerOutcome( return; } state.tracked = true; - trackAssistedQueryOutcome({ - dataset: params.dataset, - organizationSlug: params.organizationSlug, - translation: state.translation, + trackAiQueryOutcome({ + dataset: AI_QUERY_DATASETS[params.dataset], + mode: state.translation.mode, + organization: { + slug: params.organizationSlug, + features: state.translation.organizationFeatures, + }, resultCount, - error, + runId: state.translation.runId, + error: + typeof error === "boolean" || + typeof error === "string" || + error instanceof Error + ? error + : true, }); } diff --git a/packages/mcp-core/src/tools/support/search-events/seer.ts b/packages/mcp-core/src/tools/support/search-events/seer.ts index 7e8903422..e21d11368 100644 --- a/packages/mcp-core/src/tools/support/search-events/seer.ts +++ b/packages/mcp-core/src/tools/support/search-events/seer.ts @@ -1,11 +1,12 @@ -import { metrics } from "@sentry/core"; +import { logger, metrics } from "@sentry/core"; import type { z } from "zod"; import type { CrossEventQueries, SentryApiService } from "../../../api-client"; import { ApiAuthenticationError, + ApiError, type SearchAgentQuerySchema, } from "../../../api-client/index"; -import { logInfo, logWarn } from "../../../telem/logging"; +import { logWarn } from "../../../telem/logging"; import { normalizeEventsDataset, type PublicEventsDataset, @@ -51,88 +52,88 @@ export interface SeerSearchTranslation { warning?: string; // Identifies the run in the assisted query outcome metric. runId: string; - mode: AssistedQueryMode; + mode: AiQueryMode; organizationFeatures: string[]; } -// Seer doesn't return results, so the caller reports whether the translated -// query found anything. Mirrors `trackAiQueryOutcome` in Sentry's frontend, -// which reports the same metric with an `assisted_query.search_bar.*` referrer. -// See `static/app/components/searchQueryBuilder/askSeerCombobox/utils.ts`. -const ASSISTED_QUERY_OUTCOME = "assisted_query.outcome"; +export type AiQueryMode = "samples" | "aggregate"; + +export type AiQueryDataset = "spans" | "errors" | "logs" | "tracemetrics"; -// The frontend's dataset and referrer names for each Seer dataset. -const ASSISTED_QUERY_DATASETS = { +// The dataset names Sentry's frontend reports for each Seer dataset. +export const AI_QUERY_DATASETS = { errors: "errors", logs: "logs", spans: "spans", metrics: "tracemetrics", -} as const satisfies Record; - -const ASSISTED_QUERY_REFERRERS = { - errors: "assisted_query.mcp.errors", - logs: "assisted_query.mcp.logs", - spans: "assisted_query.mcp.traces", - metrics: "assisted_query.mcp.tracemetrics", -} as const satisfies Record; - -export type AssistedQueryMode = "samples" | "aggregate"; +} as const satisfies Record; -export type AssistedQueryOutcome = - | "has_results" - | "empty_results" - | "error_on_load"; +function extractErrorReason(err: Error): string { + if (err instanceof ApiError && err.detail) { + return err.detail; + } + return err.message; +} -/** - * Reports whether a Seer-translated query returned results once executed. - * Pass `error` when the query failed to run. - */ -export function trackAssistedQueryOutcome({ +// Seer doesn't return results, so the caller reports whether the translated +// query found anything. Mirrors `trackAiQueryOutcome` in Sentry's frontend +// (`static/app/components/searchQueryBuilder/askSeerCombobox/utils.ts`), which +// reports the same metric with an `assisted_query.search_bar.*` referrer. +export function trackAiQueryOutcome({ dataset, - organizationSlug, - translation, + mode, + organization, resultCount, - error, + runId, + error = false, }: { - dataset: SeerSearchDataset; - organizationSlug: string; - translation: SeerSearchTranslation; + dataset: AiQueryDataset; + mode: AiQueryMode; + organization: { slug: string; features: string[] }; resultCount: number; - error?: unknown; + runId: string; + error?: string | boolean | Error; }): void { - const outcome: AssistedQueryOutcome = error + const outcome = error ? "error_on_load" : resultCount > 0 ? "has_results" : "empty_results"; - const errorReason = error - ? error instanceof Error - ? error.message - : String(error) - : undefined; + const errorReason = + typeof error === "string" + ? error + : error instanceof Error + ? extractErrorReason(error) + : undefined; + + const referrer = `assisted_query.mcp.${dataset === "spans" ? "traces" : dataset}`; + // Seer runs are stored as conversations in the `sentry` org. + const conversationUrl = `https://sentry.io/organizations/sentry/explore/agents/conversations/${encodeURIComponent(runId)}/`; + const codeMode = organization.features.includes( + "seer-assisted-query-codemode", + ); + const crossEventEnabled = organization.features.includes( + "seer-assisted-query-cross-event-explorer", + ); const attributes = { - dataset: ASSISTED_QUERY_DATASETS[dataset], - mode: translation.mode, - org_slug: organizationSlug, - referrer: ASSISTED_QUERY_REFERRERS[dataset], - run_id: translation.runId, - // Seer runs are stored as conversations in the `sentry` org. - conversation_url: `https://sentry.io/organizations/sentry/explore/agents/conversations/${encodeURIComponent(translation.runId)}/`, + dataset, + mode, + org_slug: organization.slug, + referrer, + run_id: runId, + conversation_url: conversationUrl, outcome, - code_mode: translation.organizationFeatures.includes( - "seer-assisted-query-codemode", - ), - cross_event_enabled: translation.organizationFeatures.includes( - "seer-assisted-query-cross-event-explorer", - ), - ...(errorReason ? { error_reason: errorReason } : {}), + error_reason: errorReason, + code_mode: codeMode, + cross_event_enabled: crossEventEnabled, }; - logInfo(ASSISTED_QUERY_OUTCOME, { - extra: { ...attributes, result_count: resultCount }, + logger.info("assisted_query.outcome", { + ...attributes, + result_count: resultCount, }); - metrics.distribution(ASSISTED_QUERY_OUTCOME, resultCount, { attributes }); + metrics.distribution("assisted_query.outcome", resultCount, { attributes }); } // Returns the org's features when Seer translation is enabled, otherwise null. From ef2ac1e9c42328a1c10ece6e7a3a2617ec1eb454 Mon Sep 17 00:00:00 2001 From: Dominik Buszowiecki Date: Fri, 23 Oct 2026 10:01:15 -0400 Subject: [PATCH 4/5] ref(search-events): Reuse shared dataset and Explore mode types Use EventsApiDataset and normalizeEventsDataset for the AI query outcome dataset instead of a Seer-specific copy, and share an ExploreMode type for the Explore page's samples/aggregate mode. Co-Authored-By: Claude Opus 5.5 --- .../src/tools/support/search-events/search.ts | 3 +-- .../src/tools/support/search-events/seer.ts | 20 +++++-------------- packages/mcp-core/src/utils/url-utils.ts | 5 ++++- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/mcp-core/src/tools/support/search-events/search.ts b/packages/mcp-core/src/tools/support/search-events/search.ts index 945470021..7ef5edfd1 100644 --- a/packages/mcp-core/src/tools/support/search-events/search.ts +++ b/packages/mcp-core/src/tools/support/search-events/search.ts @@ -47,7 +47,6 @@ import { isValidReplaySort, } from "./replays"; import { - AI_QUERY_DATASETS, isSeerSearchDataset, type SeerSearchTranslation, trackAiQueryOutcome, @@ -635,7 +634,7 @@ function trackSeerOutcome( } state.tracked = true; trackAiQueryOutcome({ - dataset: AI_QUERY_DATASETS[params.dataset], + dataset: normalizeEventsDataset(params.dataset), mode: state.translation.mode, organization: { slug: params.organizationSlug, diff --git a/packages/mcp-core/src/tools/support/search-events/seer.ts b/packages/mcp-core/src/tools/support/search-events/seer.ts index e21d11368..0f9ae8f81 100644 --- a/packages/mcp-core/src/tools/support/search-events/seer.ts +++ b/packages/mcp-core/src/tools/support/search-events/seer.ts @@ -8,9 +8,11 @@ import { } from "../../../api-client/index"; import { logWarn } from "../../../telem/logging"; import { + type EventsApiDataset, normalizeEventsDataset, type PublicEventsDataset, } from "../../../utils/events-datasets"; +import type { ExploreMode } from "../../../utils/url-utils"; import { RECOMMENDED_FIELDS } from "./config"; export const SEER_SEARCH_AGENT_POLLING_INTERVAL = 1000; // 1 second @@ -52,22 +54,10 @@ export interface SeerSearchTranslation { warning?: string; // Identifies the run in the assisted query outcome metric. runId: string; - mode: AiQueryMode; + mode: ExploreMode; organizationFeatures: string[]; } -export type AiQueryMode = "samples" | "aggregate"; - -export type AiQueryDataset = "spans" | "errors" | "logs" | "tracemetrics"; - -// The dataset names Sentry's frontend reports for each Seer dataset. -export const AI_QUERY_DATASETS = { - errors: "errors", - logs: "logs", - spans: "spans", - metrics: "tracemetrics", -} as const satisfies Record; - function extractErrorReason(err: Error): string { if (err instanceof ApiError && err.detail) { return err.detail; @@ -87,8 +77,8 @@ export function trackAiQueryOutcome({ runId, error = false, }: { - dataset: AiQueryDataset; - mode: AiQueryMode; + dataset: EventsApiDataset; + mode: ExploreMode; organization: { slug: string; features: string[] }; resultCount: number; runId: string; diff --git a/packages/mcp-core/src/utils/url-utils.ts b/packages/mcp-core/src/utils/url-utils.ts index 09d8b0b0a..fa38980d0 100644 --- a/packages/mcp-core/src/utils/url-utils.ts +++ b/packages/mcp-core/src/utils/url-utils.ts @@ -11,6 +11,9 @@ export function isPublicSentryHost(host: string): boolean { return isSentryHost(host) && !host.endsWith(".my.sentry.io"); } +// The Explore page's `mode` URL parameter. +export type ExploreMode = "aggregate" | "samples"; + export interface TraceMetricIdentifier { name: string; type: string; @@ -152,7 +155,7 @@ function dedupeTraceMetrics( function buildMetricQueryState(params: { metric: TraceMetricIdentifier; query: string; - mode: "aggregate" | "samples"; + mode: ExploreMode; yAxes: string[]; groupByFields?: string[]; aggregateSortBys?: Array<{ field: string; kind: "asc" | "desc" }>; From 0c2429f9e03176f6f40c88f7e299c91010e02039 Mon Sep 17 00:00:00 2001 From: Dominik Buszowiecki Date: Fri, 23 Oct 2026 10:12:14 -0400 Subject: [PATCH 5/5] test(search-events): Trim AI query outcome tests Keep coverage for the reported attributes, failures after translation, and skipping embedded agent queries. Co-Authored-By: Claude Opus 5.5 --- .../src/tools/catalog/search-events.test.ts | 92 ------------------- 1 file changed, 92 deletions(-) diff --git a/packages/mcp-core/src/tools/catalog/search-events.test.ts b/packages/mcp-core/src/tools/catalog/search-events.test.ts index 3332ef5be..e3e378862 100644 --- a/packages/mcp-core/src/tools/catalog/search-events.test.ts +++ b/packages/mcp-core/src/tools/catalog/search-events.test.ts @@ -4711,43 +4711,6 @@ describe("search_events", () => { ); }); - it("should report empty_results when the Seer query returns no rows", async () => { - mswServer.use(...mockSeerEvents(() => HttpResponse.json({ data: [] }))); - - await searchEvents.handler(seerParams, context); - - expect(mockDistribution).toHaveBeenCalledWith( - "assisted_query.outcome", - 0, - { - attributes: expect.objectContaining({ outcome: "empty_results" }), - }, - ); - }); - - it("should report error_on_load when the Seer query fails", async () => { - mswServer.use( - ...mockSeerEvents(() => - HttpResponse.json({ detail: "Invalid query" }, { status: 400 }), - ), - ); - - await expect( - searchEvents.handler(seerParams, context), - ).rejects.toThrow(); - - expect(mockDistribution).toHaveBeenCalledWith( - "assisted_query.outcome", - 0, - { - attributes: expect.objectContaining({ - outcome: "error_on_load", - error_reason: expect.stringContaining("Invalid query"), - }), - }, - ); - }); - it("should report error_on_load when validating the Seer query fails", async () => { mswServer.use( ...mockSeerEvents(() => HttpResponse.json({ data: [] })), @@ -4778,61 +4741,6 @@ describe("search_events", () => { ); }); - it("should count non-empty buckets for a Seer time series", async () => { - mswServer.use( - mockOrganization(["mcp-search-events-seer-translate"]), - mockSeerState({ - status: "completed", - final_response: { - responses: [ - { - ...seerQuery, - query: "", - group_by: [], - visualization: [ - { chart_type: 1, y_axes: ["count()"], interval: "1d" }, - ], - sort: "-count()", - stats_period: "7d", - }, - ], - unsupported_reason: null, - }, - }), - http.get( - "https://sentry.io/api/0/organizations/test-org/events-timeseries/", - () => - HttpResponse.json({ - timeSeries: [ - { - yAxis: "count()", - values: [ - { timestamp: 1757548800000, value: 0, incomplete: false }, - { timestamp: 1757635200000, value: 8, incomplete: false }, - { - timestamp: 1757721600000, - value: null, - incomplete: false, - }, - ], - meta: { interval: 86400000 }, - }, - ], - }), - ), - ); - - await searchEvents.handler(seerParams, context); - - expect(mockDistribution).toHaveBeenCalledWith( - "assisted_query.outcome", - 1, - { - attributes: expect.objectContaining({ outcome: "has_results" }), - }, - ); - }); - it("should not report when the embedded agent translates the query", async () => { mockGenerateText.mockResolvedValueOnce( mockAIResponse("spans", "span.op:http.client"),