Surface missing credential run failures

.changeset/bright-runs-report.md

@@ -0,0 +1,5 @@
+---
+"@agent-native/core": patch
+---
+
+Surface missing LLM credentials and earlier stream errors as failed agent runs instead of successful completions.

packages/core/src/agent/run-manager.spec.ts

@@ -1,6 +1,9 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
-import { LLM_MISSING_CREDENTIALS_MESSAGE } from "./engine/credential-errors.js";
+import {
+  LLM_MISSING_CREDENTIALS_ERROR_CODE,
+  LLM_MISSING_CREDENTIALS_MESSAGE,
+} from "./engine/credential-errors.js";
 import { EngineError } from "./engine/types.js";
 import type { AgentChatEvent } from "./types.js";
 
@@ -625,6 +628,47 @@ describe("run manager soft timeout", () => {
     });
   });
 
+  it("persists missing credential terminal events as errored runs", async () => {
+    const events: AgentChatEvent[] = [];
+    const run = startRun(
+      "run-missing-credential-terminal",
+      "thread-missing-credential-terminal",
+      async (send) => {
+        send({ type: "missing_api_key" });
+      },
+      undefined,
+      { softTimeoutMs: 0 },
+    );
+    run.subscribers.add((event) => events.push(event.event));
+
+    await vi.waitFor(() =>
+      expect(updateRunStatusIfRunning).toHaveBeenCalledWith(
+        "run-missing-credential-terminal",
+        "errored",
+      ),
+    );
+
+    expect(updateRunStatusIfRunning).not.toHaveBeenCalledWith(
+      "run-missing-credential-terminal",
+      "completed",
+    );
+    expect(insertRunEvent).toHaveBeenCalledWith(
+      "run-missing-credential-terminal",
+      0,
+      JSON.stringify({ type: "missing_api_key" }),
+    );
+    expect(setRunTerminalReason).toHaveBeenCalledWith(
+      "run-missing-credential-terminal",
+      "missing_api_key",
+    );
+    expect(setRunError).toHaveBeenCalledWith(
+      "run-missing-credential-terminal",
+      LLM_MISSING_CREDENTIALS_ERROR_CODE,
+      LLM_MISSING_CREDENTIALS_MESSAGE,
+    );
+    expect(events).toContainEqual({ type: "missing_api_key" });
+  });
+
   it("maps exhausted provider 429s to a terminal rate-limit error code", async () => {
     const events: AgentChatEvent[] = [];
 

packages/core/src/agent/run-manager.ts

@@ -1,5 +1,9 @@
 import { captureError } from "../server/capture-error.js";
-import { isLlmCredentialError } from "./engine/credential-errors.js";
+import {
+  isLlmCredentialError,
+  LLM_MISSING_CREDENTIALS_ERROR_CODE,
+  LLM_MISSING_CREDENTIALS_MESSAGE,
+} from "./engine/credential-errors.js";
 import { EngineError } from "./engine/types.js";
 import {
   insertRun,
@@ -376,6 +380,10 @@ function isTerminalRunEvent(event: AgentChatEvent): boolean {
   );
 }
 
+function terminalEventForcesErroredStatus(event: AgentChatEvent | null) {
+  return event?.type === "error" || event?.type === "missing_api_key";
+}
+
 function terminalReasonForRun(
   finalStatus: "completed" | "errored" | "aborted",
   terminalEvent: AgentChatEvent | null,
@@ -920,15 +928,18 @@ export function startRun(
       // 2. Compute final status. If the completion callback threw, we'd
       //    rather mark the run errored than claim success with incomplete
       //    thread_data.
+      const terminalEvent = pendingTerminalEvent?.event ?? null;
       const finalStatus =
         run.status === "aborted"
           ? "aborted"
-          : run.status === "errored" || completionError
+          : run.status === "errored" ||
+              completionError ||
+              terminalEventForcesErroredStatus(terminalEvent)
             ? "errored"
             : "completed";
       const terminalReason = terminalReasonForRun(
         finalStatus,
-        pendingTerminalEvent?.event ?? null,
+        terminalEvent,
         run.abortReason,
         completionError,
       );
@@ -952,7 +963,8 @@ export function startRun(
         const terminalEvent: AgentChatEvent =
           finalStatus === "completed"
             ? (pendingTerminalEvent?.event ?? { type: "done" })
-            : pendingTerminalEvent?.event.type === "error"
+            : pendingTerminalEvent?.event.type === "error" ||
+                pendingTerminalEvent?.event.type === "missing_api_key"
               ? pendingTerminalEvent.event
               : pendingTerminalEvent?.event.type === "auto_continue"
                 ? // The run was checkpointed at a soft-timeout/loop boundary and
@@ -1029,14 +1041,21 @@ export function startRun(
       if (finalStatus === "errored") {
         let errorCode: string | undefined;
         let errorDetail: string | undefined;
-        for (let i = run.events.length - 1; i >= 0; i--) {
-          const ev = run.events[i].event as {
+        const diagnosticEvents = pendingTerminalEvent
+          ? [...run.events, pendingTerminalEvent]
+          : run.events;
+        for (let i = diagnosticEvents.length - 1; i >= 0; i--) {
+          const ev = diagnosticEvents[i].event as {
             type: string;
             error?: string;
             errorCode?: string;
             details?: string;
           };
-          if (ev.type === "error") {
+          if (ev.type === "missing_api_key") {
+            errorCode = LLM_MISSING_CREDENTIALS_ERROR_CODE;
+            errorDetail = LLM_MISSING_CREDENTIALS_MESSAGE;
+            break;
+          } else if (ev.type === "error") {
             errorCode = ev.errorCode;
             errorDetail = ev.error ?? ev.details;
             break;

packages/core/src/agent/run-store.spec.ts

@@ -38,7 +38,7 @@ const mockDb = {
     execCalls.push({ sql: rawSql, args });
 
     if (
-      /SELECT seq, event_data(?:, event_at)? FROM agent_run_events/i.test(
+      /SELECT seq,\s*event_data(?:,\s*event_at)?\s+FROM agent_run_events/i.test(
         rawSql,
       )
     ) {
@@ -372,6 +372,125 @@ describe("run store", () => {
     ).toBe(false);
   });
 
+  it("reconciles missing credential terminal events as errored runs", async () => {
+    latestEventRows = [
+      {
+        seq: 9,
+        event_at: 123_456,
+        event_data: JSON.stringify({ type: "missing_api_key" }),
+      },
+    ];
+
+    const reaped = await reapIfStale("run-missing-key-event");
+
+    expect(reaped).toBe(false);
+    const repair = execCalls.find(
+      (call) =>
+        /UPDATE agent_runs/i.test(call.sql) &&
+        /SET status = \?/i.test(call.sql),
+    );
+    expect(repair?.args[0]).toBe("errored");
+    expect(repair?.args[1]).toBe(123_456);
+    expect(repair?.args[2]).toBe("missing_credentials");
+    expect(repair?.args[3]).toEqual(
+      expect.stringContaining("No LLM provider is connected"),
+    );
+    expect(repair?.args[4]).toBe("missing_api_key");
+    expect(repair?.args[5]).toBe("run-missing-key-event");
+  });
+
+  it("keeps an earlier stream error from reconciling as a later successful terminal event", async () => {
+    latestEventRows = [
+      {
+        seq: 10,
+        event_at: 124_000,
+        event_data: JSON.stringify({ type: "done" }),
+      },
+      {
+        seq: 9,
+        event_at: 123_456,
+        event_data: JSON.stringify({
+          type: "error",
+          errorCode: "provider_failed",
+          error: "Provider failed",
+          details: "model returned 500",
+        }),
+      },
+    ];
+
+    const reaped = await reapIfStale("run-error-then-done");
+
+    expect(reaped).toBe(false);
+    const repair = execCalls.find(
+      (call) =>
+        /UPDATE agent_runs/i.test(call.sql) &&
+        /SET status = \?/i.test(call.sql),
+    );
+    expect(repair?.args[0]).toBe("errored");
+    expect(repair?.args[1]).toBe(123_456);
+    expect(repair?.args[2]).toBe("provider_failed");
+    expect(repair?.args[3]).toBe("model returned 500");
+    expect(repair?.args[4]).toBe("error:provider_failed");
+    expect(repair?.args[5]).toBe("run-error-then-done");
+  });
+
+  it("keeps an earlier missing credential event from reconciling as a later successful terminal event", async () => {
+    latestEventRows = [
+      {
+        seq: 10,
+        event_at: 124_000,
+        event_data: JSON.stringify({ type: "done" }),
+      },
+      {
+        seq: 9,
+        event_at: 123_456,
+        event_data: JSON.stringify({ type: "missing_api_key" }),
+      },
+    ];
+
+    await reapIfStale("run-missing-then-done");
+
+    const repair = execCalls.find(
+      (call) =>
+        /UPDATE agent_runs/i.test(call.sql) &&
+        /SET status = \?/i.test(call.sql),
+    );
+    expect(repair?.args[0]).toBe("errored");
+    expect(repair?.args[1]).toBe(123_456);
+    expect(repair?.args[2]).toBe("missing_credentials");
+    expect(repair?.args[4]).toBe("missing_api_key");
+    expect(repair?.args[5]).toBe("run-missing-then-done");
+  });
+
+  it("still repairs a synthetic stale-run event when a later done event was persisted", async () => {
+    latestEventRows = [
+      {
+        seq: 10,
+        event_at: 124_000,
+        event_data: JSON.stringify({ type: "done" }),
+      },
+      {
+        seq: 9,
+        event_at: 123_456,
+        event_data: JSON.stringify(STALE_RUN_ERROR_EVENT),
+      },
+    ];
+
+    await reapIfStale("run-stale-then-done");
+
+    const repair = execCalls.find(
+      (call) =>
+        /UPDATE agent_runs/i.test(call.sql) &&
+        /SET status = \?/i.test(call.sql),
+    );
+    expect(repair?.args[0]).toBe("completed");
+    expect(repair?.args[1]).toBe(124_000);
+    expect(repair?.args[2]).toBeNull();
+    expect(repair?.args[3]).toBeNull();
+    expect(repair?.args[4]).toBe("done");
+    expect(repair?.args[5]).toBe("run-stale-then-done");
+  });
+
   it("reconciles legacy terminal events without stamping repair time", async () => {
     latestEventRows = [
       { seq: 9, event_data: JSON.stringify({ type: "done" }) },