diff --git a/packages/mcp-core/src/tools/catalog/search-events.test.ts b/packages/mcp-core/src/tools/catalog/search-events.test.ts index df8615d6a..e3e378862 100644 --- a/packages/mcp-core/src/tools/catalog/search-events.test.ts +++ b/packages/mcp-core/src/tools/catalog/search-events.test.ts @@ -1,3 +1,4 @@ +import { metrics } from "@sentry/core"; import { mswServer } from "@sentry/mcp-server-mocks"; import { APICallError, generateText } from "ai"; import { HttpResponse, http } from "msw"; @@ -15,6 +16,14 @@ vi.mock("@ai-sdk/openai", () => { }; }); +vi.mock("@sentry/core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + metrics: { ...actual.metrics, distribution: vi.fn() }, + }; +}); + vi.mock("ai", async (importOriginal) => { const actual = await importOriginal(); return { @@ -4004,8 +4013,11 @@ describe("search_events", () => { () => HttpResponse.json({ session, sentry_run_id: "run-uuid" }), ); + const mockDistribution = vi.mocked(metrics.distribution); + beforeEach(() => { mockSeerStart.mockClear(); + mockDistribution.mockClear(); mswServer.use( mockProject, http.post( @@ -4645,5 +4657,107 @@ describe("search_events", () => { expect(mockGenerateText).toHaveBeenCalled(); }); + + describe("assisted query outcome", () => { + const mockSeerEvents = ( + resolver: Parameters[1], + ): Parameters => [ + mockOrganization([ + "mcp-search-events-seer-translate", + "seer-assisted-query-cross-event-explorer", + ]), + mockSeerState({ + status: "completed", + final_response: { responses: [seerQuery], unsupported_reason: null }, + }), + http.get( + "https://sentry.io/api/0/organizations/test-org/events/", + resolver, + ), + ]; + + it("should report has_results when the Seer query returns rows", async () => { + mswServer.use( + ...mockSeerEvents(() => + HttpResponse.json({ + data: [ + { "span.description": "GET /a", "p95(span.duration)": 1 }, + { "span.description": "GET /b", "p95(span.duration)": 2 }, + ], + }), + ), + ); + + await searchEvents.handler(seerParams, context); + + expect(mockDistribution).toHaveBeenCalledTimes(1); + expect(mockDistribution).toHaveBeenCalledWith( + "assisted_query.outcome", + 2, + { + attributes: { + dataset: "spans", + mode: "aggregate", + org_slug: "test-org", + referrer: "assisted_query.mcp.traces", + run_id: "run-uuid", + conversation_url: + "https://sentry.io/organizations/sentry/explore/agents/conversations/run-uuid/", + outcome: "has_results", + code_mode: false, + cross_event_enabled: true, + }, + }, + ); + }); + + it("should report error_on_load when validating the Seer query fails", async () => { + mswServer.use( + ...mockSeerEvents(() => HttpResponse.json({ data: [] })), + http.get( + "https://sentry.io/api/0/organizations/test-org/events/validate/", + () => + HttpResponse.json( + { detail: "Validation broke" }, + { status: 400 }, + ), + ), + ); + + await expect( + searchEvents.handler(seerParams, context), + ).rejects.toThrow(); + + expect(mockDistribution).toHaveBeenCalledTimes(1); + expect(mockDistribution).toHaveBeenCalledWith( + "assisted_query.outcome", + 0, + { + attributes: expect.objectContaining({ + outcome: "error_on_load", + error_reason: expect.any(String), + }), + }, + ); + }); + + it("should not report when the embedded agent translates the query", async () => { + mockGenerateText.mockResolvedValueOnce( + mockAIResponse("spans", "span.op:http.client"), + ); + mswServer.use( + mockOrganization([]), + http.get( + "https://sentry.io/api/0/organizations/test-org/events/", + () => HttpResponse.json({ data: [] }), + ), + ); + + await searchEvents.handler(seerParams, context); + + expect(mockGenerateText).toHaveBeenCalled(); + expect(mockDistribution).not.toHaveBeenCalled(); + }); + }); }); }); diff --git a/packages/mcp-core/src/tools/support/search-events/search.ts b/packages/mcp-core/src/tools/support/search-events/search.ts index 07c4e729e..c8aaa5910 100644 --- a/packages/mcp-core/src/tools/support/search-events/search.ts +++ b/packages/mcp-core/src/tools/support/search-events/search.ts @@ -48,7 +48,12 @@ import { formatReplayResults, isValidReplaySort, } from "./replays"; -import { isSeerSearchDataset, translateWithSeer } from "./seer"; +import { + isSeerSearchDataset, + type SeerSearchTranslation, + trackAiQueryOutcome, + translateWithSeer, +} from "./seer"; import { formatEventsValidationResults, isAggregateQuery, @@ -596,10 +601,65 @@ export function logSearchEventsError( /** * Shared handler for search_events and the dataset-specific search tools. */ +interface SeerOutcomeState { + translation: SeerSearchTranslation | null; + tracked: boolean; +} + +// Only Seer's translations are tracked, not the embedded agent's, and at most +// once per run. +function trackSeerOutcome( + state: SeerOutcomeState, + params: SearchEventsParams, + resultCount: number, + error: unknown = false, +): void { + if ( + !state.translation || + state.tracked || + !isSeerSearchDataset(params.dataset) + ) { + return; + } + state.tracked = true; + trackAiQueryOutcome({ + dataset: normalizeEventsDataset(params.dataset), + mode: state.translation.mode, + organization: { + slug: params.organizationSlug, + features: state.translation.organizationFeatures, + }, + resultCount, + runId: state.translation.runId, + error: + typeof error === "boolean" || + typeof error === "string" || + error instanceof Error + ? error + : true, + }); +} + export async function runSearchEvents( params: SearchEventsParams, context: ServerContext, options: RunSearchEventsOptions = {}, +): Promise { + // Any failure after Seer translates the query counts as `error_on_load`. + const seerOutcome: SeerOutcomeState = { translation: null, tracked: false }; + try { + return await executeSearchEvents(params, context, options, seerOutcome); + } catch (error) { + trackSeerOutcome(seerOutcome, params, 0, error); + throw error; + } +} + +async function executeSearchEvents( + params: SearchEventsParams, + context: ServerContext, + options: RunSearchEventsOptions, + seerOutcome: SeerOutcomeState, ): Promise { const lockDataset = options.lockDataset === true; const apiService = apiServiceFromContext(context, { @@ -676,6 +736,7 @@ export async function runSearchEvents( query: params.query, }) : null; + seerOutcome.translation = seerTranslation; if (seerTranslation && !projectId) { projectId = "-1"; } @@ -948,6 +1009,14 @@ export async function runSearchEvents( dataset, ...timeParams, }); + // A series always has buckets, so count the ones with data. + trackSeerOutcome( + seerOutcome, + params, + series.timeSeries + .flatMap((s) => s.values) + .filter((point) => point.value != null && point.value !== 0).length, + ); const statsUrl = apiService.getEventsExplorerUrl( organizationSlug, timeSeriesQuery, @@ -1064,6 +1133,7 @@ export async function runSearchEvents( "gen_ai.tool.call.result.count", eventData.length, ); + trackSeerOutcome(seerOutcome, params, eventData.length); const conversationId = extractConversationIdFromSearchQuery(sentryQuery); const explorerUrl = conversationId diff --git a/packages/mcp-core/src/tools/support/search-events/seer.ts b/packages/mcp-core/src/tools/support/search-events/seer.ts index 704f80d20..9de875ed2 100644 --- a/packages/mcp-core/src/tools/support/search-events/seer.ts +++ b/packages/mcp-core/src/tools/support/search-events/seer.ts @@ -1,16 +1,20 @@ +import { logger, metrics } from "@sentry/core"; import { isAggregateFieldOrEquation } from "@sentry/toolkit-core/events-fields"; import { getSortField } from "@sentry/toolkit-core/events-sort"; import type { z } from "zod"; import type { CrossEventQueries, SentryApiService } from "../../../api-client"; import { ApiAuthenticationError, + ApiError, type SearchAgentQuerySchema, } from "../../../api-client/index"; import { logWarn } from "../../../telem/logging"; import { + type EventsApiDataset, normalizeEventsDataset, type PublicEventsDataset, } from "../../../utils/events-datasets"; +import type { ExploreMode } from "../../../utils/url-utils"; import { RECOMMENDED_FIELDS } from "./config"; export const SEER_SEARCH_AGENT_POLLING_INTERVAL = 1000; // 1 second @@ -50,28 +54,102 @@ export interface SeerSearchTranslation { explanation: string; // Correctness warnings must be shown even when explanations are disabled. warning?: string; + // Identifies the run in the assisted query outcome metric. + runId: string; + mode: ExploreMode; + organizationFeatures: string[]; } -async function hasSeerSearchAgentAccess( +function extractErrorReason(err: Error): string { + if (err instanceof ApiError && err.detail) { + return err.detail; + } + return err.message; +} + +// Seer doesn't return results, so the caller reports whether the translated +// query found anything. Mirrors `trackAiQueryOutcome` in Sentry's frontend +// (`static/app/components/searchQueryBuilder/askSeerCombobox/utils.ts`), which +// reports the same metric with an `assisted_query.search_bar.*` referrer. +export function trackAiQueryOutcome({ + dataset, + mode, + organization, + resultCount, + runId, + error = false, +}: { + dataset: EventsApiDataset; + mode: ExploreMode; + organization: { slug: string; features: string[] }; + resultCount: number; + runId: string; + error?: string | boolean | Error; +}): void { + const outcome = error + ? "error_on_load" + : resultCount > 0 + ? "has_results" + : "empty_results"; + const errorReason = + typeof error === "string" + ? error + : error instanceof Error + ? extractErrorReason(error) + : undefined; + + const referrer = `assisted_query.mcp.${dataset === "spans" ? "traces" : dataset}`; + // Seer runs are stored as conversations in the `sentry` org. + const conversationUrl = `https://sentry.io/organizations/sentry/explore/agents/conversations/${encodeURIComponent(runId)}/`; + const codeMode = organization.features.includes( + "seer-assisted-query-codemode", + ); + const crossEventEnabled = organization.features.includes( + "seer-assisted-query-cross-event-explorer", + ); + + const attributes = { + dataset, + mode, + org_slug: organization.slug, + referrer, + run_id: runId, + conversation_url: conversationUrl, + outcome, + error_reason: errorReason, + code_mode: codeMode, + cross_event_enabled: crossEventEnabled, + }; + + logger.info("assisted_query.outcome", { + ...attributes, + result_count: resultCount, + }); + metrics.distribution("assisted_query.outcome", resultCount, { attributes }); +} + +// Returns the org's features when Seer translation is enabled, otherwise null. +async function getSeerSearchAgentFeatures( apiService: SentryApiService, organizationSlug: string, -): Promise { +): Promise { // Sentry omits `features` unless explicitly requested. const organization = await apiService.getOrganization(organizationSlug, { includeFeatureFlags: true, detailed: false, }); if (organization.hideAiFeatures) { - return false; + return null; } const features = organization.features ?? []; - return features.includes(SEER_TRANSLATE_FEATURE); + return features.includes(SEER_TRANSLATE_FEATURE) ? features : null; } function toSearchTranslation( result: z.output, dataset: SeerSearchDataset, suggestedProjectIds: number[], + run: { runId: string; organizationFeatures: string[] }, ): SeerSearchTranslation { const aggregates = result.visualization.flatMap((chart) => chart.y_axes); const fields = @@ -160,6 +238,9 @@ function toSearchTranslation( suggestedProjectIds, explanation, warning, + runId: run.runId, + mode: result.mode === "aggregates" ? "aggregate" : "samples", + organizationFeatures: run.organizationFeatures, }; } @@ -184,7 +265,11 @@ export async function translateWithSeer({ query: string; }): Promise { try { - if (!(await hasSeerSearchAgentAccess(apiService, organizationSlug))) { + const organizationFeatures = await getSeerSearchAgentFeatures( + apiService, + organizationSlug, + ); + if (!organizationFeatures) { return null; } @@ -215,7 +300,10 @@ export async function translateWithSeer({ (id) => id !== Number(projectId), ) : []; - return toSearchTranslation(result, dataset, suggestedProjectIds); + return toSearchTranslation(result, dataset, suggestedProjectIds, { + runId: run.sentry_run_id, + organizationFeatures, + }); } if (session?.status === "error") { return null; diff --git a/packages/mcp-core/src/utils/url-utils.ts b/packages/mcp-core/src/utils/url-utils.ts index 60262bdf2..41bb1f64e 100644 --- a/packages/mcp-core/src/utils/url-utils.ts +++ b/packages/mcp-core/src/utils/url-utils.ts @@ -12,6 +12,9 @@ export function isPublicSentryHost(host: string): boolean { return isSentryHost(host) && !host.endsWith(".my.sentry.io"); } +// The Explore page's `mode` URL parameter. +export type ExploreMode = "aggregate" | "samples"; + export interface TraceMetricIdentifier { name: string; type: string; @@ -153,7 +156,7 @@ function dedupeTraceMetrics( function buildMetricQueryState(params: { metric: TraceMetricIdentifier; query: string; - mode: "aggregate" | "samples"; + mode: ExploreMode; yAxes: string[]; groupByFields?: string[]; aggregateSortBys?: Array<{ field: string; kind: "asc" | "desc" }>;