feat(search-events): Track assisted query outcome for Seer translations

packages/mcp-core/src/tools/catalog/search-events.test.ts

@@ -1,3 +1,4 @@
+import { metrics } from "@sentry/core";
 import { mswServer } from "@sentry/mcp-server-mocks";
 import { APICallError, generateText } from "ai";
 import { HttpResponse, http } from "msw";
@@ -15,6 +16,14 @@ vi.mock("@ai-sdk/openai", () => {
   };
 });
 
+vi.mock("@sentry/core", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("@sentry/core")>();
+  return {
+    ...actual,
+    metrics: { ...actual.metrics, distribution: vi.fn() },
+  };
+});
+
 vi.mock("ai", async (importOriginal) => {
   const actual = await importOriginal<typeof import("ai")>();
   return {
@@ -4004,8 +4013,11 @@ describe("search_events", () => {
         () => HttpResponse.json({ session, sentry_run_id: "run-uuid" }),
       );
 
+    const mockDistribution = vi.mocked(metrics.distribution);
+
     beforeEach(() => {
       mockSeerStart.mockClear();
+      mockDistribution.mockClear();
       mswServer.use(
         mockProject,
         http.post(
@@ -4645,5 +4657,107 @@ describe("search_events", () => {
 
       expect(mockGenerateText).toHaveBeenCalled();
     });
+
+    describe("assisted query outcome", () => {
+      const mockSeerEvents = (
+        resolver: Parameters<typeof http.get>[1],
+      ): Parameters<typeof mswServer.use> => [
+        mockOrganization([
+          "mcp-search-events-seer-translate",
+          "seer-assisted-query-cross-event-explorer",
+        ]),
+        mockSeerState({
+          status: "completed",
+          final_response: { responses: [seerQuery], unsupported_reason: null },
+        }),
+        http.get(
+          "https://sentry.io/api/0/organizations/test-org/events/",
+          resolver,
+        ),
+      ];
+
+      it("should report has_results when the Seer query returns rows", async () => {
+        mswServer.use(
+          ...mockSeerEvents(() =>
+            HttpResponse.json({
+              data: [
+                { "span.description": "GET /a", "p95(span.duration)": 1 },
+                { "span.description": "GET /b", "p95(span.duration)": 2 },
+              ],
+            }),
+          ),
+        );
+
+        await searchEvents.handler(seerParams, context);
+
+        expect(mockDistribution).toHaveBeenCalledTimes(1);
+        expect(mockDistribution).toHaveBeenCalledWith(
+          "assisted_query.outcome",
+          2,
+          {
+            attributes: {
+              dataset: "spans",
+              mode: "aggregate",
+              org_slug: "test-org",
+              referrer: "assisted_query.mcp.traces",
+              run_id: "run-uuid",
+              conversation_url:
+                "https://sentry.io/organizations/sentry/explore/agents/conversations/run-uuid/",
+              outcome: "has_results",
+              code_mode: false,
+              cross_event_enabled: true,
+            },
+          },
+        );
+      });
+
+      it("should report error_on_load when validating the Seer query fails", async () => {
+        mswServer.use(
+          ...mockSeerEvents(() => HttpResponse.json({ data: [] })),
+          http.get(
+            "https://sentry.io/api/0/organizations/test-org/events/validate/",
+            () =>
+              HttpResponse.json(
+                { detail: "Validation broke" },
+                { status: 400 },
+              ),
+          ),
+        );
+
+        await expect(
+          searchEvents.handler(seerParams, context),
+        ).rejects.toThrow();
+
+        expect(mockDistribution).toHaveBeenCalledTimes(1);
+        expect(mockDistribution).toHaveBeenCalledWith(
+          "assisted_query.outcome",
+          0,
+          {
+            attributes: expect.objectContaining({
+              outcome: "error_on_load",
+              error_reason: expect.any(String),
+            }),
+          },
+        );
+      });
+
+      it("should not report when the embedded agent translates the query", async () => {
+        mockGenerateText.mockResolvedValueOnce(
+          mockAIResponse("spans", "span.op:http.client"),
+        );
+        mswServer.use(
+          mockOrganization([]),
+          http.get(
+            "https://sentry.io/api/0/organizations/test-org/events/",
+            () => HttpResponse.json({ data: [] }),
+          ),
+        );
+
+        await searchEvents.handler(seerParams, context);
+
+        expect(mockGenerateText).toHaveBeenCalled();
+        expect(mockDistribution).not.toHaveBeenCalled();
+      });
+    });
   });
 });

packages/mcp-core/src/tools/support/search-events/search.ts

@@ -48,7 +48,12 @@ import {
   formatReplayResults,
   isValidReplaySort,
 } from "./replays";
-import { isSeerSearchDataset, translateWithSeer } from "./seer";
+import {
+  isSeerSearchDataset,
+  type SeerSearchTranslation,
+  trackAiQueryOutcome,
+  translateWithSeer,
+} from "./seer";
 import {
   formatEventsValidationResults,
   isAggregateQuery,
@@ -596,10 +601,65 @@ export function logSearchEventsError(
 /**
  * Shared handler for search_events and the dataset-specific search tools.
  */
+interface SeerOutcomeState {
+  translation: SeerSearchTranslation | null;
+  tracked: boolean;
+}
+
+// Only Seer's translations are tracked, not the embedded agent's, and at most
+// once per run.
+function trackSeerOutcome(
+  state: SeerOutcomeState,
+  params: SearchEventsParams,
+  resultCount: number,
+  error: unknown = false,
+): void {
+  if (
+    !state.translation ||
+    state.tracked ||
+    !isSeerSearchDataset(params.dataset)
+  ) {
+    return;
+  }
+  state.tracked = true;
+  trackAiQueryOutcome({
+    dataset: normalizeEventsDataset(params.dataset),
+    mode: state.translation.mode,
+    organization: {
+      slug: params.organizationSlug,
+      features: state.translation.organizationFeatures,
+    },
+    resultCount,
+    runId: state.translation.runId,
+    error:
+      typeof error === "boolean" ||
+      typeof error === "string" ||
+      error instanceof Error
+        ? error
+        : true,
+  });
+}
+
 export async function runSearchEvents(
   params: SearchEventsParams,
   context: ServerContext,
   options: RunSearchEventsOptions = {},
+): Promise<string> {
+  // Any failure after Seer translates the query counts as `error_on_load`.
+  const seerOutcome: SeerOutcomeState = { translation: null, tracked: false };
+  try {
+    return await executeSearchEvents(params, context, options, seerOutcome);
+  } catch (error) {
+    trackSeerOutcome(seerOutcome, params, 0, error);
+    throw error;
+  }
+}
+
+async function executeSearchEvents(
+  params: SearchEventsParams,
+  context: ServerContext,
+  options: RunSearchEventsOptions,
+  seerOutcome: SeerOutcomeState,
 ): Promise<string> {
   const lockDataset = options.lockDataset === true;
   const apiService = apiServiceFromContext(context, {
@@ -676,6 +736,7 @@ export async function runSearchEvents(
           query: params.query,
         })
       : null;
+  seerOutcome.translation = seerTranslation;
   if (seerTranslation && !projectId) {
     projectId = "-1";
   }
@@ -948,6 +1009,14 @@ export async function runSearchEvents(
       dataset,
       ...timeParams,
     });
+    // A series always has buckets, so count the ones with data.
+    trackSeerOutcome(
+      seerOutcome,
+      params,
+      series.timeSeries
+        .flatMap((s) => s.values)
+        .filter((point) => point.value != null && point.value !== 0).length,
+    );
     const statsUrl = apiService.getEventsExplorerUrl(
       organizationSlug,
       timeSeriesQuery,
@@ -1064,6 +1133,7 @@ export async function runSearchEvents(
     "gen_ai.tool.call.result.count",
     eventData.length,
   );
+  trackSeerOutcome(seerOutcome, params, eventData.length);
 
   const conversationId = extractConversationIdFromSearchQuery(sentryQuery);
   const explorerUrl = conversationId