From 1ecce26bc9e3ec822274558c3b5b5415aab1c34c Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:14:07 -0400 Subject: [PATCH] Surface missing credential run failures --- .changeset/bright-runs-report.md | 5 + packages/core/src/agent/run-manager.spec.ts | 46 ++++++- packages/core/src/agent/run-manager.ts | 33 ++++- packages/core/src/agent/run-store.spec.ts | 121 +++++++++++++++++- packages/core/src/agent/run-store.ts | 94 ++++++++++---- .../src/client/agent-chat-adapter.spec.ts | 91 +++++++++++++ .../core/src/client/agent-chat-adapter.ts | 39 ++++-- 7 files changed, 382 insertions(+), 47 deletions(-) create mode 100644 .changeset/bright-runs-report.md diff --git a/.changeset/bright-runs-report.md b/.changeset/bright-runs-report.md new file mode 100644 index 0000000000..2ddfc75ec8 --- /dev/null +++ b/.changeset/bright-runs-report.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Surface missing LLM credentials and earlier stream errors as failed agent runs instead of successful completions. diff --git a/packages/core/src/agent/run-manager.spec.ts b/packages/core/src/agent/run-manager.spec.ts index e3c7e823cc..df4b5254e9 100644 --- a/packages/core/src/agent/run-manager.spec.ts +++ b/packages/core/src/agent/run-manager.spec.ts @@ -1,6 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { LLM_MISSING_CREDENTIALS_MESSAGE } from "./engine/credential-errors.js"; +import { + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, +} from "./engine/credential-errors.js"; import { EngineError } from "./engine/types.js"; import type { AgentChatEvent } from "./types.js"; @@ -595,6 +598,47 @@ describe("run manager soft timeout", () => { }); }); + it("persists missing credential terminal events as errored runs", async () => { + const events: AgentChatEvent[] = []; + const run = startRun( + "run-missing-credential-terminal", + "thread-missing-credential-terminal", + async (send) => { + send({ type: "missing_api_key" }); + }, + undefined, + { softTimeoutMs: 0 }, + ); + run.subscribers.add((event) => events.push(event.event)); + + await vi.waitFor(() => + expect(updateRunStatusIfRunning).toHaveBeenCalledWith( + "run-missing-credential-terminal", + "errored", + ), + ); + + expect(updateRunStatusIfRunning).not.toHaveBeenCalledWith( + "run-missing-credential-terminal", + "completed", + ); + expect(insertRunEvent).toHaveBeenCalledWith( + "run-missing-credential-terminal", + 0, + JSON.stringify({ type: "missing_api_key" }), + ); + expect(setRunTerminalReason).toHaveBeenCalledWith( + "run-missing-credential-terminal", + "missing_api_key", + ); + expect(setRunError).toHaveBeenCalledWith( + "run-missing-credential-terminal", + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, + ); + expect(events).toContainEqual({ type: "missing_api_key" }); + }); + it("maps exhausted provider 429s to a terminal rate-limit error code", async () => { const events: AgentChatEvent[] = []; diff --git a/packages/core/src/agent/run-manager.ts b/packages/core/src/agent/run-manager.ts index d0ed11fa45..0a5337837a 100644 --- a/packages/core/src/agent/run-manager.ts +++ b/packages/core/src/agent/run-manager.ts @@ -1,5 +1,9 @@ import { captureError } from "../server/capture-error.js"; -import { isLlmCredentialError } from "./engine/credential-errors.js"; +import { + isLlmCredentialError, + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, +} from "./engine/credential-errors.js"; import { EngineError } from "./engine/types.js"; import { insertRun, @@ -369,6 +373,10 @@ function isTerminalRunEvent(event: AgentChatEvent): boolean { ); } +function terminalEventForcesErroredStatus(event: AgentChatEvent | null) { + return event?.type === "error" || event?.type === "missing_api_key"; +} + function terminalReasonForRun( finalStatus: "completed" | "errored" | "aborted", terminalEvent: AgentChatEvent | null, @@ -872,15 +880,18 @@ export function startRun( // 2. Compute final status. If the completion callback threw, we'd // rather mark the run errored than claim success with incomplete // thread_data. + const terminalEvent = pendingTerminalEvent?.event ?? null; const finalStatus = run.status === "aborted" ? "aborted" - : run.status === "errored" || completionError + : run.status === "errored" || + completionError || + terminalEventForcesErroredStatus(terminalEvent) ? "errored" : "completed"; const terminalReason = terminalReasonForRun( finalStatus, - pendingTerminalEvent?.event ?? null, + terminalEvent, run.abortReason, completionError, ); @@ -904,7 +915,8 @@ export function startRun( const terminalEvent: AgentChatEvent = finalStatus === "completed" ? (pendingTerminalEvent?.event ?? { type: "done" }) - : pendingTerminalEvent?.event.type === "error" + : pendingTerminalEvent?.event.type === "error" || + pendingTerminalEvent?.event.type === "missing_api_key" ? pendingTerminalEvent.event : pendingTerminalEvent?.event.type === "auto_continue" ? // The run was checkpointed at a soft-timeout/loop boundary and @@ -981,14 +993,21 @@ export function startRun( if (finalStatus === "errored") { let errorCode: string | undefined; let errorDetail: string | undefined; - for (let i = run.events.length - 1; i >= 0; i--) { - const ev = run.events[i].event as { + const diagnosticEvents = pendingTerminalEvent + ? [...run.events, pendingTerminalEvent] + : run.events; + for (let i = diagnosticEvents.length - 1; i >= 0; i--) { + const ev = diagnosticEvents[i].event as { type: string; error?: string; errorCode?: string; details?: string; }; - if (ev.type === "error") { + if (ev.type === "missing_api_key") { + errorCode = LLM_MISSING_CREDENTIALS_ERROR_CODE; + errorDetail = LLM_MISSING_CREDENTIALS_MESSAGE; + break; + } else if (ev.type === "error") { errorCode = ev.errorCode; errorDetail = ev.error ?? ev.details; break; diff --git a/packages/core/src/agent/run-store.spec.ts b/packages/core/src/agent/run-store.spec.ts index dd8bff30e7..d85bff2d63 100644 --- a/packages/core/src/agent/run-store.spec.ts +++ b/packages/core/src/agent/run-store.spec.ts @@ -38,7 +38,7 @@ const mockDb = { execCalls.push({ sql: rawSql, args }); if ( - /SELECT seq, event_data(?:, event_at)? FROM agent_run_events/i.test( + /SELECT seq,\s*event_data(?:,\s*event_at)?\s+FROM agent_run_events/i.test( rawSql, ) ) { @@ -370,6 +370,125 @@ describe("run store", () => { ).toBe(false); }); + it("reconciles missing credential terminal events as errored runs", async () => { + latestEventRows = [ + { + seq: 9, + event_at: 123_456, + event_data: JSON.stringify({ type: "missing_api_key" }), + }, + ]; + + const reaped = await reapIfStale("run-missing-key-event"); + + expect(reaped).toBe(false); + const repair = execCalls.find( + (call) => + /UPDATE agent_runs/i.test(call.sql) && + /SET status = \?/i.test(call.sql), + ); + expect(repair?.args[0]).toBe("errored"); + expect(repair?.args[1]).toBe(123_456); + expect(repair?.args[2]).toBe("missing_credentials"); + expect(repair?.args[3]).toEqual( + expect.stringContaining("No LLM provider is connected"), + ); + expect(repair?.args[4]).toBe("missing_api_key"); + expect(repair?.args[5]).toBe("run-missing-key-event"); + }); + + it("keeps an earlier stream error from reconciling as a later successful terminal event", async () => { + latestEventRows = [ + { + seq: 10, + event_at: 124_000, + event_data: JSON.stringify({ type: "done" }), + }, + { + seq: 9, + event_at: 123_456, + event_data: JSON.stringify({ + type: "error", + errorCode: "provider_failed", + error: "Provider failed", + details: "model returned 500", + }), + }, + ]; + + const reaped = await reapIfStale("run-error-then-done"); + + expect(reaped).toBe(false); + const repair = execCalls.find( + (call) => + /UPDATE agent_runs/i.test(call.sql) && + /SET status = \?/i.test(call.sql), + ); + expect(repair?.args[0]).toBe("errored"); + expect(repair?.args[1]).toBe(123_456); + expect(repair?.args[2]).toBe("provider_failed"); + expect(repair?.args[3]).toBe("model returned 500"); + expect(repair?.args[4]).toBe("error:provider_failed"); + expect(repair?.args[5]).toBe("run-error-then-done"); + }); + + it("keeps an earlier missing credential event from reconciling as a later successful terminal event", async () => { + latestEventRows = [ + { + seq: 10, + event_at: 124_000, + event_data: JSON.stringify({ type: "done" }), + }, + { + seq: 9, + event_at: 123_456, + event_data: JSON.stringify({ type: "missing_api_key" }), + }, + ]; + + await reapIfStale("run-missing-then-done"); + + const repair = execCalls.find( + (call) => + /UPDATE agent_runs/i.test(call.sql) && + /SET status = \?/i.test(call.sql), + ); + expect(repair?.args[0]).toBe("errored"); + expect(repair?.args[1]).toBe(123_456); + expect(repair?.args[2]).toBe("missing_credentials"); + expect(repair?.args[4]).toBe("missing_api_key"); + expect(repair?.args[5]).toBe("run-missing-then-done"); + }); + + it("still repairs a synthetic stale-run event when a later done event was persisted", async () => { + latestEventRows = [ + { + seq: 10, + event_at: 124_000, + event_data: JSON.stringify({ type: "done" }), + }, + { + seq: 9, + event_at: 123_456, + event_data: JSON.stringify(STALE_RUN_ERROR_EVENT), + }, + ]; + + await reapIfStale("run-stale-then-done"); + + const repair = execCalls.find( + (call) => + /UPDATE agent_runs/i.test(call.sql) && + /SET status = \?/i.test(call.sql), + ); + expect(repair?.args[0]).toBe("completed"); + expect(repair?.args[1]).toBe(124_000); + expect(repair?.args[2]).toBeNull(); + expect(repair?.args[3]).toBeNull(); + expect(repair?.args[4]).toBe("done"); + expect(repair?.args[5]).toBe("run-stale-then-done"); + }); + it("reconciles legacy terminal events without stamping repair time", async () => { latestEventRows = [ { seq: 9, event_data: JSON.stringify({ type: "done" }) }, diff --git a/packages/core/src/agent/run-store.ts b/packages/core/src/agent/run-store.ts index 7c72114cc5..1bc9465698 100644 --- a/packages/core/src/agent/run-store.ts +++ b/packages/core/src/agent/run-store.ts @@ -7,6 +7,10 @@ import { getDbExec, intType, isPostgres } from "../db/client.js"; import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js"; import { widenIntColumnsToBigInt } from "../db/widen-columns.js"; import { captureError } from "../server/capture-error.js"; +import { + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, +} from "./engine/credential-errors.js"; import type { AgentChatEvent } from "./types.js"; let _initPromise: Promise | undefined; @@ -792,9 +796,9 @@ function terminalStatusForEvent( event: AgentChatEvent, ): "completed" | "errored" | null { if (event.type === "error") return "errored"; + if (event.type === "missing_api_key") return "errored"; if ( event.type === "done" || - event.type === "missing_api_key" || event.type === "loop_limit" || event.type === "auto_continue" ) { @@ -812,32 +816,75 @@ function terminalReasonForEvent(event: AgentChatEvent): string | null { return null; } -async function getLatestRunEvent(runId: string): Promise<{ +function isRealFailureTerminalEvent(event: AgentChatEvent): boolean { + if (event.type === "missing_api_key") return true; + if (event.type !== "error") return false; + return event.errorCode !== STALE_RUN_ERROR_EVENT.errorCode; +} + +async function getRunEventForReconciliation(runId: string): Promise<{ event: AgentChatEvent; eventAt: number | null; } | null> { const client = getDbExec(); const { rows } = await client.execute({ - sql: `SELECT seq, event_data, event_at FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, + sql: `SELECT seq, event_data, event_at + FROM agent_run_events + WHERE run_id = ? + AND ( + event_data LIKE '{"type":"done"%' + OR event_data LIKE '{"type":"error"%' + OR event_data LIKE '{"type":"missing_api_key"%' + OR event_data LIKE '{"type":"loop_limit"%' + OR event_data LIKE '{"type":"auto_continue"%' + ) + ORDER BY seq DESC`, args: [runId], }); - const row = rows[0] as - | { - event_at?: number | string | null; - event_data?: string; + let latestTerminal: { + event: AgentChatEvent; + eventAt: number | null; + } | null = null; + for (const row of rows as Array<{ + event_at?: number | string | null; + event_data?: string; + }>) { + const raw = row.event_data; + if (!raw) continue; + try { + const event = JSON.parse(raw) as AgentChatEvent; + if (!terminalStatusForEvent(event) || !terminalReasonForEvent(event)) { + continue; } - | undefined; - const raw = row?.event_data; - if (!raw) return null; - try { - const eventAt = row.event_at == null ? NaN : Number(row.event_at); - return { - event: JSON.parse(raw) as AgentChatEvent, - eventAt: Number.isFinite(eventAt) && eventAt > 0 ? eventAt : null, - }; - } catch { - return null; + const rawEventAt = row.event_at == null ? NaN : Number(row.event_at); + const parsed = { + event, + eventAt: + Number.isFinite(rawEventAt) && rawEventAt > 0 ? rawEventAt : null, + }; + if (!latestTerminal) latestTerminal = parsed; + // A real stream failure must not be laundered into success by a later + // done/continuation boundary. Keep the synthetic stale-run error special: + // that repair marker may be superseded by a later durable done event. + if (isRealFailureTerminalEvent(event)) return parsed; + } catch { + continue; + } } + return latestTerminal; +} + +function errorCodeForTerminalEvent(event: AgentChatEvent): string | null { + if (event.type === "missing_api_key") + return LLM_MISSING_CREDENTIALS_ERROR_CODE; + if (event.type === "error") return event.errorCode ?? null; + return null; +} + +function errorDetailForTerminalEvent(event: AgentChatEvent): string | null { + if (event.type === "missing_api_key") return LLM_MISSING_CREDENTIALS_MESSAGE; + if (event.type !== "error") return null; + return (event.details || event.error || "").slice(0, 2000) || null; } /** @@ -853,20 +900,15 @@ export async function reconcileTerminalRunFromEvents( runId: string, ): Promise { await ensureRunTables(); - const latest = await getLatestRunEvent(runId); + const latest = await getRunEventForReconciliation(runId); if (!latest) return false; const status = terminalStatusForEvent(latest.event); const terminalReason = terminalReasonForEvent(latest.event); if (!status || !terminalReason) return false; const client = getDbExec(); - const errorCode = - latest.event.type === "error" ? (latest.event.errorCode ?? null) : null; - const errorDetail = - latest.event.type === "error" - ? (latest.event.details || latest.event.error || "").slice(0, 2000) || - null - : null; + const errorCode = errorCodeForTerminalEvent(latest.event); + const errorDetail = errorDetailForTerminalEvent(latest.event); const { rowsAffected } = await client.execute({ sql: `UPDATE agent_runs SET status = ?, diff --git a/packages/core/src/client/agent-chat-adapter.spec.ts b/packages/core/src/client/agent-chat-adapter.spec.ts index 6b2941cf72..35db489685 100644 --- a/packages/core/src/client/agent-chat-adapter.spec.ts +++ b/packages/core/src/client/agent-chat-adapter.spec.ts @@ -5234,6 +5234,97 @@ describe("createAgentChatAdapter", () => { expect(last.content.at(-1).text).toContain("Working and done"); }); + it("surfaces missing credentials from a terminal background run instead of completing successfully", async () => { + vi.useFakeTimers(); + const dispatchEvent = vi.fn(); + vi.stubGlobal("window", { dispatchEvent }); + vi.stubGlobal( + "CustomEvent", + class CustomEvent { + type: string; + detail: unknown; + constructor(type: string, init?: { detail?: unknown }) { + this.type = type; + this.detail = init?.detail; + } + }, + ); + + let postCount = 0; + const fetchSpy = vi.fn(async (url: string, init?: RequestInit) => { + if (url === "/_agent-native/agent-chat" && init?.method === "POST") { + postCount += 1; + return backgroundSseResponse( + [ + { type: "text", text: "Checking credentials" }, + { type: "auto_continue", reason: "run_timeout" }, + ], + "run-bg-missing-key", + ); + } + if (url.includes("/runs/active")) { + return jsonResponse({ + active: true, + runId: "run-bg-missing-key", + threadId: "thread-bg-missing-key", + status: "completed", + terminalReason: "missing_api_key", + dispatchMode: "background-processing", + heartbeatAt: Date.now(), + lastProgressAt: Date.now(), + }); + } + return jsonResponse({ error: "unexpected" }, 500); + }); + vi.stubGlobal("fetch", fetchSpy); + + const adapter = createAgentChatAdapter({ + apiUrl: "/_agent-native/agent-chat", + tabId: "chat-bg-missing-key", + threadId: "thread-bg-missing-key", + }); + const promise = drain( + adapter.run({ + messages: [ + { + role: "user", + content: [{ type: "text", text: "answer my analytics question" }], + }, + ], + abortSignal: new AbortController().signal, + } as any), + ); + + await vi.advanceTimersByTimeAsync(2_000); + const results = await promise; + + expect(postCount).toBe(1); + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "agent-chat:missing-api-key", + detail: { + tabId: "chat-bg-missing-key", + threadId: "thread-bg-missing-key", + }, + }), + ); + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "agent-chat:run-error", + detail: expect.objectContaining({ + errorCode: "missing_credentials", + message: expect.stringContaining("No LLM provider is connected"), + }), + }), + ); + const last = results.at(-1) as any; + expect(last.status).toEqual({ type: "incomplete", reason: "error" }); + expect(last.metadata?.custom?.runError?.errorCode).toBe( + "missing_credentials", + ); + expect(last.content.at(-1).text).toContain("No LLM provider is connected"); + }); + it("surfaces a terminal error when a background run goes idle with no successor", async () => { // If the server-chained successor never appears (lost handoff that even // the server sweep failed to resurface), the follow loop must end the diff --git a/packages/core/src/client/agent-chat-adapter.ts b/packages/core/src/client/agent-chat-adapter.ts index ca6bb7b641..d5168199c1 100644 --- a/packages/core/src/client/agent-chat-adapter.ts +++ b/packages/core/src/client/agent-chat-adapter.ts @@ -1,6 +1,10 @@ import type { ChatModelAdapter, ChatModelRunResult } from "@assistant-ui/react"; import { actionPreparationContinuationNote } from "../agent/action-continuation-guidance.js"; +import { + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, +} from "../agent/engine/credential-errors.js"; import type { AgentChatStructuredContentPart, AgentChatStructuredMessage, @@ -158,7 +162,9 @@ const BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS = 150_000; * (`agent_runs.terminal_reason`, surfaced by /runs/active). These runs died in * server-side handoff machinery where no richer error event may exist, so the * client owns the message. Keys are matched after stripping an optional - * `error:` prefix (`terminalReasonForEvent` records `error:`). + * `error:` prefix (`terminalReasonForEvent` records `error:`). Keep this + * map scoped to terminal failures, including reasons that must override a + * mistakenly completed row. */ const BACKGROUND_TERMINAL_REASON_MESSAGES: Record = { background_worker_never_started: @@ -167,6 +173,7 @@ const BACKGROUND_TERMINAL_REASON_MESSAGES: Record = { "The agent's background worker could not hand off the next step of this run. Retry to continue from the preserved context.", dispatch_payload_missing: "The agent's background run lost its saved request data and could not continue. Retry to start a fresh run.", + missing_api_key: LLM_MISSING_CREDENTIALS_MESSAGE, turn_continuation_budget_exhausted: "This request needed more automatic continuations than allowed and was stopped. Try breaking it into smaller steps.", }; @@ -2076,6 +2083,9 @@ export function createAgentChatAdapter( ...(runId ? { runId } : {}), }; if (typeof window !== "undefined") { + if (args.errorCode === LLM_MISSING_CREDENTIALS_ERROR_CODE) { + dispatchMissingApiKey(); + } window.dispatchEvent( new CustomEvent("agent-chat:run-error", { detail: { ...runError, tabId }, @@ -2114,6 +2124,22 @@ export function createAgentChatAdapter( // terminal_reason is either a bare reason ("dispatch_payload_missing") // or "error:" when derived from a terminal error event. const terminalReason = rawTerminalReason.replace(/^error:/, ""); + const mappedMessage = terminalReason + ? BACKGROUND_TERMINAL_REASON_MESSAGES[terminalReason] + : undefined; + const mappedErrorCode = + terminalReason === "missing_api_key" + ? LLM_MISSING_CREDENTIALS_ERROR_CODE + : terminalReason; + + if (mappedMessage) { + yield* emitBackgroundTerminalError({ + message: mappedMessage, + errorCode: mappedErrorCode, + details: `terminal_reason: ${rawTerminalReason}`, + }); + return; + } if (status === "completed") { // The turn finished server-side; the events we managed to fold are @@ -2130,17 +2156,6 @@ export function createAgentChatAdapter( return; } - const mappedMessage = terminalReason - ? BACKGROUND_TERMINAL_REASON_MESSAGES[terminalReason] - : undefined; - if (mappedMessage) { - yield* emitBackgroundTerminalError({ - message: mappedMessage, - errorCode: terminalReason, - details: `terminal_reason: ${rawTerminalReason}`, - }); - return; - } if (lastRecoverableRunError) { // A replayed terminal error event already carried the real // message (recoverable errors replay as auto-continue signals