diff --git a/.changeset/bright-runs-report.md b/.changeset/bright-runs-report.md new file mode 100644 index 0000000000..2ddfc75ec8 --- /dev/null +++ b/.changeset/bright-runs-report.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Surface missing LLM credentials and earlier stream errors as failed agent runs instead of successful completions. diff --git a/packages/core/src/agent/production-agent.chain-continuation.spec.ts b/packages/core/src/agent/production-agent.chain-continuation.spec.ts index 4c314c2841..d758604dd0 100644 --- a/packages/core/src/agent/production-agent.chain-continuation.spec.ts +++ b/packages/core/src/agent/production-agent.chain-continuation.spec.ts @@ -64,6 +64,17 @@ function timeoutBoundaryRun(): ActiveRun { return makeRun([{ type: "auto_continue", reason: "run_timeout" }]); } +function recoverableErrorBoundaryRun(): ActiveRun { + return makeRun([ + { + type: "error", + error: "Provider connection failed", + errorCode: "provider_failed", + recoverable: true, + }, + ]); +} + interface Harness { deps: Required< Pick< @@ -181,12 +192,31 @@ describe("chainServerDrivenContinuation — transactional handoff (foreground se // The chunk is marked terminal ONLY after the handoff landed. expect(h.deps.markBackgroundContinuationChunkTerminal).toHaveBeenCalledWith( - { runId: "run-chunk0", continuationReason: "run_timeout" }, + { + runId: "run-chunk0", + continuationReason: "run_timeout", + terminalEvent: { type: "auto_continue", reason: "run_timeout" }, + }, ); // No failure path was taken. expect(h.deps.updateRunStatusIfRunning).not.toHaveBeenCalled(); }); + it("passes a recoverable error boundary to the chunk terminal marker", async () => { + const h = makeHarness(); + const run = recoverableErrorBoundaryRun(); + + await runChain(h, { run }); + + expect(h.deps.markBackgroundContinuationChunkTerminal).toHaveBeenCalledWith( + { + runId: "run-chunk0", + continuationReason: expect.any(String), + terminalEvent: run.events.at(-1)?.event, + }, + ); + }); + it("dispatches IDS ONLY (payloadRef marker) — never the chat body — and fully awaits the response", async () => { const h = makeHarness(); await runChain(h); diff --git a/packages/core/src/agent/production-agent.spec.ts b/packages/core/src/agent/production-agent.spec.ts index 69b9d35014..026dc6664d 100644 --- a/packages/core/src/agent/production-agent.spec.ts +++ b/packages/core/src/agent/production-agent.spec.ts @@ -6531,13 +6531,14 @@ describe("shouldChainBackgroundContinuation (server-driven background chain)", ( it("marks a successfully chained background chunk terminal before the worker returns", async () => { const updateRunStatusIfRunning = vi.fn(async () => true); + const setRunError = vi.fn(async () => {}); const setRunTerminalReason = vi.fn(async () => {}); await expect( markBackgroundContinuationChunkTerminal({ runId: "run-old", continuationReason: "no_progress", - deps: { updateRunStatusIfRunning, setRunTerminalReason }, + deps: { updateRunStatusIfRunning, setRunError, setRunTerminalReason }, }), ).resolves.toBe(true); @@ -6546,17 +6547,54 @@ describe("shouldChainBackgroundContinuation (server-driven background chain)", ( "completed", ); expect(setRunTerminalReason).toHaveBeenCalledWith("run-old", "no_progress"); + expect(setRunError).not.toHaveBeenCalled(); + }); + + it("marks a recoverable error continuation chunk errored with its durable failure details", async () => { + const updateRunStatusIfRunning = vi.fn(async () => true); + const setRunError = vi.fn(async () => {}); + const setRunTerminalReason = vi.fn(async () => {}); + + await expect( + markBackgroundContinuationChunkTerminal({ + runId: "run-error-boundary", + continuationReason: "run_timeout", + terminalEvent: { + type: "error", + error: "Provider connection failed", + errorCode: "provider_failed", + details: "upstream returned 500", + recoverable: true, + }, + deps: { updateRunStatusIfRunning, setRunError, setRunTerminalReason }, + }), + ).resolves.toBe(true); + + expect(updateRunStatusIfRunning).toHaveBeenCalledWith( + "run-error-boundary", + "errored", + ); + expect(setRunTerminalReason).toHaveBeenCalledWith( + "run-error-boundary", + "error:provider_failed", + ); + expect(setRunError).toHaveBeenCalledWith( + "run-error-boundary", + "provider_failed", + "upstream returned 500", + ); }); it("does not overwrite terminal reason when another process already finished the chunk", async () => { const updateRunStatusIfRunning = vi.fn(async () => false); + const setRunError = vi.fn(async () => {}); const setRunTerminalReason = vi.fn(async () => {}); await expect( markBackgroundContinuationChunkTerminal({ runId: "run-old", continuationReason: "run_timeout", - deps: { updateRunStatusIfRunning, setRunTerminalReason }, + deps: { updateRunStatusIfRunning, setRunError, setRunTerminalReason }, }), ).resolves.toBe(false); @@ -6565,6 +6603,7 @@ describe("shouldChainBackgroundContinuation (server-driven background chain)", ( "completed", ); expect(setRunTerminalReason).not.toHaveBeenCalled(); + expect(setRunError).not.toHaveBeenCalled(); }); }); diff --git a/packages/core/src/agent/production-agent.ts b/packages/core/src/agent/production-agent.ts index 3ed65d8f7e..ca559fbee6 100644 --- a/packages/core/src/agent/production-agent.ts +++ b/packages/core/src/agent/production-agent.ts @@ -136,6 +136,7 @@ import { insertRun, updateRunHeartbeat, updateRunStatusIfRunning, + setRunError, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, @@ -4881,18 +4882,47 @@ export function shouldChainBackgroundContinuation(opts: { export async function markBackgroundContinuationChunkTerminal(opts: { runId: string; continuationReason: AgentLoopContinuationReason; + terminalEvent?: AgentChatEvent; deps?: { updateRunStatusIfRunning?: typeof updateRunStatusIfRunning; + setRunError?: typeof setRunError; setRunTerminalReason?: typeof setRunTerminalReason; }; }): Promise { const updateStatus = opts.deps?.updateRunStatusIfRunning ?? updateRunStatusIfRunning; + const persistError = opts.deps?.setRunError ?? setRunError; const setTerminalReason = opts.deps?.setRunTerminalReason ?? setRunTerminalReason; - const updated = await updateStatus(opts.runId, "completed"); + const terminalEvent = opts.terminalEvent; + const isTerminalFailure = + terminalEvent?.type === "error" || + terminalEvent?.type === "missing_api_key"; + const terminalReason = + terminalEvent?.type === "error" + ? `error:${terminalEvent.errorCode || "unknown"}` + : terminalEvent?.type === "missing_api_key" + ? "missing_api_key" + : opts.continuationReason; + const updated = await updateStatus( + opts.runId, + isTerminalFailure ? "errored" : "completed", + ); if (updated) { - await setTerminalReason(opts.runId, opts.continuationReason); + await setTerminalReason(opts.runId, terminalReason); + if (terminalEvent?.type === "error") { + await persistError( + opts.runId, + terminalEvent.errorCode, + terminalEvent.details || terminalEvent.error, + ); + } else if (terminalEvent?.type === "missing_api_key") { + await persistError( + opts.runId, + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, + ); + } } return updated; } @@ -5513,6 +5543,7 @@ export async function chainServerDrivenContinuation(opts: { .markBackgroundContinuationChunkTerminal({ runId, continuationReason, + terminalEvent: run.events.at(-1)?.event, }) .catch(() => {}); } catch (chainErr) { diff --git a/packages/core/src/agent/run-manager.spec.ts b/packages/core/src/agent/run-manager.spec.ts index d4864cf394..f5009754d6 100644 --- a/packages/core/src/agent/run-manager.spec.ts +++ b/packages/core/src/agent/run-manager.spec.ts @@ -1,6 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { LLM_MISSING_CREDENTIALS_MESSAGE } from "./engine/credential-errors.js"; +import { + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, +} from "./engine/credential-errors.js"; import { EngineError } from "./engine/types.js"; import type { AgentChatEvent } from "./types.js"; @@ -632,6 +635,90 @@ describe("run manager soft timeout", () => { }); }); + it("persists missing credential terminal events as errored runs", async () => { + const events: AgentChatEvent[] = []; + const onComplete = vi.fn(async () => {}); + const run = startRun( + "run-missing-credential-terminal", + "thread-missing-credential-terminal", + async (send) => { + send({ type: "missing_api_key" }); + }, + onComplete, + { softTimeoutMs: 0 }, + ); + run.subscribers.add((event) => events.push(event.event)); + + await vi.waitFor(() => + expect(updateRunStatusIfRunning).toHaveBeenCalledWith( + "run-missing-credential-terminal", + "errored", + ), + ); + + expect(updateRunStatusIfRunning).not.toHaveBeenCalledWith( + "run-missing-credential-terminal", + "completed", + ); + expect(insertRunEvent).toHaveBeenCalledWith( + "run-missing-credential-terminal", + 0, + JSON.stringify({ type: "missing_api_key" }), + ); + expect(setRunTerminalReason).toHaveBeenCalledWith( + "run-missing-credential-terminal", + "missing_api_key", + ); + expect(setRunError).toHaveBeenCalledWith( + "run-missing-credential-terminal", + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, + ); + expect(events).toContainEqual({ type: "missing_api_key" }); + expect(onComplete).toHaveBeenCalledWith( + expect.objectContaining({ + status: "errored", + events: [ + expect.objectContaining({ event: { type: "missing_api_key" } }), + ], + }), + ); + }); + + it("passes an emitted terminal error to completion callbacks as errored", async () => { + const onComplete = vi.fn(async () => {}); + + startRun( + "run-error-terminal-callback", + "thread-error-terminal-callback", + async (send) => { + send({ + type: "error", + error: "Provider failed", + errorCode: "provider_failed", + recoverable: true, + }); + }, + onComplete, + { softTimeoutMs: 0 }, + ); + + await vi.waitFor(() => expect(onComplete).toHaveBeenCalledTimes(1)); + expect(onComplete).toHaveBeenCalledWith( + expect.objectContaining({ + status: "errored", + events: [ + expect.objectContaining({ + event: expect.objectContaining({ + type: "error", + errorCode: "provider_failed", + }), + }), + ], + }), + ); + }); + it("maps exhausted provider 429s to a terminal rate-limit error code", async () => { const events: AgentChatEvent[] = []; diff --git a/packages/core/src/agent/run-manager.ts b/packages/core/src/agent/run-manager.ts index 81fccdcbee..88a93ceba8 100644 --- a/packages/core/src/agent/run-manager.ts +++ b/packages/core/src/agent/run-manager.ts @@ -1,5 +1,9 @@ import { captureError } from "../server/capture-error.js"; -import { isLlmCredentialError } from "./engine/credential-errors.js"; +import { + isLlmCredentialError, + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, +} from "./engine/credential-errors.js"; import { EngineError } from "./engine/types.js"; import { insertRun, @@ -377,6 +381,10 @@ function isTerminalRunEvent(event: AgentChatEvent): boolean { ); } +function terminalEventForcesErroredStatus(event: AgentChatEvent | null) { + return event?.type === "error" || event?.type === "missing_api_key"; +} + function terminalReasonForRun( finalStatus: "completed" | "errored" | "aborted", terminalEvent: AgentChatEvent | null, @@ -899,14 +907,27 @@ export function startRun( // /runs/active check while we wait for SQL writes to land. let completionError: unknown = null; let terminalPersistenceError: unknown = null; + const terminalEvent = pendingTerminalEvent?.event ?? null; if ( onComplete && !(run.status === "aborted" && run.abortReason === "no_progress") ) { try { - const completionRun: ActiveRun = pendingTerminalEvent - ? { ...run, events: [...run.events, pendingTerminalEvent] } - : run; + const completionStatus = + run.status !== "aborted" && + terminalEventForcesErroredStatus(terminalEvent) + ? "errored" + : run.status; + const completionRun: ActiveRun = + pendingTerminalEvent || completionStatus !== run.status + ? { + ...run, + status: completionStatus, + events: pendingTerminalEvent + ? [...run.events, pendingTerminalEvent] + : run.events, + } + : run; await onComplete(completionRun); } catch (err) { completionError = err; @@ -924,12 +945,14 @@ export function startRun( const finalStatus = run.status === "aborted" ? "aborted" - : run.status === "errored" || completionError + : run.status === "errored" || + completionError || + terminalEventForcesErroredStatus(terminalEvent) ? "errored" : "completed"; const terminalReason = terminalReasonForRun( finalStatus, - pendingTerminalEvent?.event ?? null, + terminalEvent, run.abortReason, completionError, ); @@ -953,7 +976,8 @@ export function startRun( const terminalEvent: AgentChatEvent = finalStatus === "completed" ? (pendingTerminalEvent?.event ?? { type: "done" }) - : pendingTerminalEvent?.event.type === "error" + : pendingTerminalEvent?.event.type === "error" || + pendingTerminalEvent?.event.type === "missing_api_key" ? pendingTerminalEvent.event : pendingTerminalEvent?.event.type === "auto_continue" ? // The run was checkpointed at a soft-timeout/loop boundary and @@ -1030,14 +1054,21 @@ export function startRun( if (finalStatus === "errored") { let errorCode: string | undefined; let errorDetail: string | undefined; - for (let i = run.events.length - 1; i >= 0; i--) { - const ev = run.events[i].event as { + const diagnosticEvents = pendingTerminalEvent + ? [...run.events, pendingTerminalEvent] + : run.events; + for (let i = diagnosticEvents.length - 1; i >= 0; i--) { + const ev = diagnosticEvents[i].event as { type: string; error?: string; errorCode?: string; details?: string; }; - if (ev.type === "error") { + if (ev.type === "missing_api_key") { + errorCode = LLM_MISSING_CREDENTIALS_ERROR_CODE; + errorDetail = LLM_MISSING_CREDENTIALS_MESSAGE; + break; + } else if (ev.type === "error") { errorCode = ev.errorCode; errorDetail = ev.error ?? ev.details; break; diff --git a/packages/core/src/agent/run-store.spec.ts b/packages/core/src/agent/run-store.spec.ts index 44c11a82ce..b58dfaca4c 100644 --- a/packages/core/src/agent/run-store.spec.ts +++ b/packages/core/src/agent/run-store.spec.ts @@ -42,7 +42,7 @@ const mockDb = { execCalls.push({ sql: rawSql, args }); if ( - /SELECT seq, event_data(?:, event_at)? FROM agent_run_events/i.test( + /SELECT seq,\s*event_data(?:,\s*event_at)?\s+FROM agent_run_events/i.test( rawSql, ) ) { @@ -392,6 +392,130 @@ describe("run store", () => { ).toBe(false); }); + it("reconciles missing credential terminal events as errored runs", async () => { + latestEventRows = [ + { + seq: 9, + event_at: 123_456, + event_data: JSON.stringify({ type: "missing_api_key" }), + }, + ]; + + const reaped = await reapIfStale("run-missing-key-event"); + + expect(reaped).toBe(false); + const repair = execCalls.find( + (call) => + /UPDATE agent_runs/i.test(call.sql) && + /SET status = \?/i.test(call.sql), + ); + expect(repair?.args[0]).toBe("errored"); + expect(repair?.args[1]).toBe(123_456); + expect(repair?.args[2]).toBe("missing_credentials"); + expect(repair?.args[3]).toEqual( + expect.stringContaining("No LLM provider is connected"), + ); + expect(repair?.args[4]).toBe("missing_api_key"); + expect(repair?.args[5]).toBe("run-missing-key-event"); + }); + + it("keeps an earlier stream error from reconciling as a later successful terminal event", async () => { + latestEventRows = [ + { + seq: 10, + event_at: 124_000, + event_data: JSON.stringify({ type: "done" }), + }, + { + seq: 9, + event_at: 123_456, + event_data: JSON.stringify({ + type: "error", + errorCode: "provider_failed", + error: "Provider failed", + details: "model returned 500", + }), + }, + ]; + + const reaped = await reapIfStale("run-error-then-done"); + + expect(reaped).toBe(false); + const repair = execCalls.find( + (call) => + /UPDATE agent_runs/i.test(call.sql) && + /SET status = \?/i.test(call.sql), + ); + expect(repair?.args[0]).toBe("errored"); + expect(repair?.args[1]).toBe(123_456); + expect(repair?.args[2]).toBe("provider_failed"); + expect(repair?.args[3]).toBe("model returned 500"); + expect(repair?.args[4]).toBe("error:provider_failed"); + expect(repair?.args[5]).toBe("run-error-then-done"); + const eventLookup = execCalls.find((call) => + /SELECT seq, event_data, event_at/i.test(call.sql), + ); + expect(eventLookup?.sql).toMatch(/ORDER BY seq DESC\s+LIMIT \?/i); + expect(eventLookup?.args).toEqual(["run-error-then-done", 100]); + }); + + it("keeps an earlier missing credential event from reconciling as a later successful terminal event", async () => { + latestEventRows = [ + { + seq: 10, + event_at: 124_000, + event_data: JSON.stringify({ type: "done" }), + }, + { + seq: 9, + event_at: 123_456, + event_data: JSON.stringify({ type: "missing_api_key" }), + }, + ]; + + await reapIfStale("run-missing-then-done"); + + const repair = execCalls.find( + (call) => + /UPDATE agent_runs/i.test(call.sql) && + /SET status = \?/i.test(call.sql), + ); + expect(repair?.args[0]).toBe("errored"); + expect(repair?.args[1]).toBe(123_456); + expect(repair?.args[2]).toBe("missing_credentials"); + expect(repair?.args[4]).toBe("missing_api_key"); + expect(repair?.args[5]).toBe("run-missing-then-done"); + }); + + it("still repairs a synthetic stale-run event when a later done event was persisted", async () => { + latestEventRows = [ + { + seq: 10, + event_at: 124_000, + event_data: JSON.stringify({ type: "done" }), + }, + { + seq: 9, + event_at: 123_456, + event_data: JSON.stringify(STALE_RUN_ERROR_EVENT), + }, + ]; + + await reapIfStale("run-stale-then-done"); + + const repair = execCalls.find( + (call) => + /UPDATE agent_runs/i.test(call.sql) && + /SET status = \?/i.test(call.sql), + ); + expect(repair?.args[0]).toBe("completed"); + expect(repair?.args[1]).toBe(124_000); + expect(repair?.args[2]).toBeNull(); + expect(repair?.args[3]).toBeNull(); + expect(repair?.args[4]).toBe("done"); + expect(repair?.args[5]).toBe("run-stale-then-done"); + }); + it("reconciles legacy terminal events without stamping repair time", async () => { latestEventRows = [ { seq: 9, event_data: JSON.stringify({ type: "done" }) }, diff --git a/packages/core/src/agent/run-store.ts b/packages/core/src/agent/run-store.ts index 6f6f125142..000d986967 100644 --- a/packages/core/src/agent/run-store.ts +++ b/packages/core/src/agent/run-store.ts @@ -7,6 +7,10 @@ import { getDbExec, intType, isPostgres } from "../db/client.js"; import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js"; import { widenIntColumnsToBigInt } from "../db/widen-columns.js"; import { captureError } from "../server/capture-error.js"; +import { + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, +} from "./engine/credential-errors.js"; import type { AgentChatEvent } from "./types.js"; let _initPromise: Promise | undefined; @@ -877,9 +881,9 @@ function terminalStatusForEvent( event: AgentChatEvent, ): "completed" | "errored" | null { if (event.type === "error") return "errored"; + if (event.type === "missing_api_key") return "errored"; if ( event.type === "done" || - event.type === "missing_api_key" || event.type === "loop_limit" || event.type === "auto_continue" ) { @@ -897,32 +901,78 @@ function terminalReasonForEvent(event: AgentChatEvent): string | null { return null; } -async function getLatestRunEvent(runId: string): Promise<{ +function isRealFailureTerminalEvent(event: AgentChatEvent): boolean { + if (event.type === "missing_api_key") return true; + if (event.type !== "error") return false; + return event.errorCode !== STALE_RUN_ERROR_EVENT.errorCode; +} + +const RUN_RECONCILIATION_TERMINAL_EVENT_LIMIT = 100; + +async function getRunEventForReconciliation(runId: string): Promise<{ event: AgentChatEvent; eventAt: number | null; } | null> { const client = getDbExec(); const { rows } = await client.execute({ - sql: `SELECT seq, event_data, event_at FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, - args: [runId], + sql: `SELECT seq, event_data, event_at + FROM agent_run_events + WHERE run_id = ? + AND ( + event_data LIKE '{"type":"done"%' + OR event_data LIKE '{"type":"error"%' + OR event_data LIKE '{"type":"missing_api_key"%' + OR event_data LIKE '{"type":"loop_limit"%' + OR event_data LIKE '{"type":"auto_continue"%' + ) + ORDER BY seq DESC + LIMIT ?`, + args: [runId, RUN_RECONCILIATION_TERMINAL_EVENT_LIMIT], }); - const row = rows[0] as - | { - event_at?: number | string | null; - event_data?: string; + let latestTerminal: { + event: AgentChatEvent; + eventAt: number | null; + } | null = null; + for (const row of rows as Array<{ + event_at?: number | string | null; + event_data?: string; + }>) { + const raw = row.event_data; + if (!raw) continue; + try { + const event = JSON.parse(raw) as AgentChatEvent; + if (!terminalStatusForEvent(event) || !terminalReasonForEvent(event)) { + continue; } - | undefined; - const raw = row?.event_data; - if (!raw) return null; - try { - const eventAt = row.event_at == null ? NaN : Number(row.event_at); - return { - event: JSON.parse(raw) as AgentChatEvent, - eventAt: Number.isFinite(eventAt) && eventAt > 0 ? eventAt : null, - }; - } catch { - return null; + const rawEventAt = row.event_at == null ? NaN : Number(row.event_at); + const parsed = { + event, + eventAt: + Number.isFinite(rawEventAt) && rawEventAt > 0 ? rawEventAt : null, + }; + if (!latestTerminal) latestTerminal = parsed; + // A real stream failure must not be laundered into success by a later + // done/continuation boundary. Keep the synthetic stale-run error special: + // that repair marker may be superseded by a later durable done event. + if (isRealFailureTerminalEvent(event)) return parsed; + } catch { + continue; + } } + return latestTerminal; +} + +function errorCodeForTerminalEvent(event: AgentChatEvent): string | null { + if (event.type === "missing_api_key") + return LLM_MISSING_CREDENTIALS_ERROR_CODE; + if (event.type === "error") return event.errorCode ?? null; + return null; +} + +function errorDetailForTerminalEvent(event: AgentChatEvent): string | null { + if (event.type === "missing_api_key") return LLM_MISSING_CREDENTIALS_MESSAGE; + if (event.type !== "error") return null; + return (event.details || event.error || "").slice(0, 2000) || null; } /** @@ -938,20 +988,15 @@ export async function reconcileTerminalRunFromEvents( runId: string, ): Promise { await ensureRunTables(); - const latest = await getLatestRunEvent(runId); + const latest = await getRunEventForReconciliation(runId); if (!latest) return false; const status = terminalStatusForEvent(latest.event); const terminalReason = terminalReasonForEvent(latest.event); if (!status || !terminalReason) return false; const client = getDbExec(); - const errorCode = - latest.event.type === "error" ? (latest.event.errorCode ?? null) : null; - const errorDetail = - latest.event.type === "error" - ? (latest.event.details || latest.event.error || "").slice(0, 2000) || - null - : null; + const errorCode = errorCodeForTerminalEvent(latest.event); + const errorDetail = errorDetailForTerminalEvent(latest.event); const { rowsAffected } = await client.execute({ sql: `UPDATE agent_runs SET status = ?, diff --git a/packages/core/src/client/agent-chat-adapter.spec.ts b/packages/core/src/client/agent-chat-adapter.spec.ts index 63ab0d8f71..0e8f859444 100644 --- a/packages/core/src/client/agent-chat-adapter.spec.ts +++ b/packages/core/src/client/agent-chat-adapter.spec.ts @@ -5263,6 +5263,146 @@ describe("createAgentChatAdapter", () => { expect(last.content.at(-1).text).toContain("Working and done"); }); + it.each([ + { + terminalReason: "missing_api_key", + expectedErrorCode: "missing_credentials", + expectedMessage: "No LLM provider is connected", + dispatchesMissingKey: true, + activeRunId: "run-bg-missing-key", + initialEvents: undefined, + }, + { + terminalReason: "error:missing_credentials", + expectedErrorCode: "missing_credentials", + expectedMessage: "No LLM provider is connected", + dispatchesMissingKey: true, + activeRunId: "run-bg-missing-key", + initialEvents: undefined, + }, + { + terminalReason: "error:provider_failed", + expectedErrorCode: "provider_failed", + expectedMessage: "background run failed", + dispatchesMissingKey: false, + activeRunId: "run-bg-provider-failure", + initialEvents: [ + { type: "text", text: "Checking credentials" }, + { + type: "error", + error: "An earlier chunk timed out", + errorCode: "old_chunk_timeout", + recoverable: true, + }, + ], + }, + ])( + "surfaces $terminalReason from a terminal background run instead of completing successfully", + async ({ + terminalReason, + expectedErrorCode, + expectedMessage, + dispatchesMissingKey, + activeRunId, + initialEvents, + }) => { + vi.useFakeTimers(); + const dispatchEvent = vi.fn(); + vi.stubGlobal("window", { dispatchEvent }); + vi.stubGlobal( + "CustomEvent", + class CustomEvent { + type: string; + detail: unknown; + constructor(type: string, init?: { detail?: unknown }) { + this.type = type; + this.detail = init?.detail; + } + }, + ); + + let postCount = 0; + const fetchSpy = vi.fn(async (url: string, init?: RequestInit) => { + if (url === "/_agent-native/agent-chat" && init?.method === "POST") { + postCount += 1; + return backgroundSseResponse( + initialEvents ?? [ + { type: "text", text: "Checking credentials" }, + { type: "auto_continue", reason: "run_timeout" }, + ], + "run-bg-missing-key", + ); + } + if (url.includes("/runs/active")) { + return jsonResponse({ + active: true, + runId: activeRunId, + threadId: "thread-bg-missing-key", + status: "completed", + terminalReason, + dispatchMode: "background-processing", + heartbeatAt: Date.now(), + lastProgressAt: Date.now(), + }); + } + return jsonResponse({ error: "unexpected" }, 500); + }); + vi.stubGlobal("fetch", fetchSpy); + + const adapter = createAgentChatAdapter({ + apiUrl: "/_agent-native/agent-chat", + tabId: "chat-bg-missing-key", + threadId: "thread-bg-missing-key", + }); + const promise = drain( + adapter.run({ + messages: [ + { + role: "user", + content: [{ type: "text", text: "answer my analytics question" }], + }, + ], + abortSignal: new AbortController().signal, + } as any), + ); + + await vi.advanceTimersByTimeAsync(2_000); + const results = await promise; + + expect(postCount).toBe(1); + const missingKeyEvent = expect.objectContaining({ + type: "agent-chat:missing-api-key", + detail: { + tabId: "chat-bg-missing-key", + threadId: "thread-bg-missing-key", + }, + }); + if (dispatchesMissingKey) { + expect(dispatchEvent).toHaveBeenCalledWith(missingKeyEvent); + } else { + expect(dispatchEvent).not.toHaveBeenCalledWith(missingKeyEvent); + } + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "agent-chat:run-error", + detail: expect.objectContaining({ + errorCode: expectedErrorCode, + message: expect.stringContaining(expectedMessage), + }), + }), + ); + const last = results.at(-1) as any; + expect(last.status).toEqual({ type: "incomplete", reason: "error" }); + expect(last.metadata?.custom?.runError?.errorCode).toBe( + expectedErrorCode, + ); + expect(last.content.at(-1).text).toContain(expectedMessage); + expect(last.content.at(-1).text).not.toContain( + "An earlier chunk timed out", + ); + }, + ); + it("self-POSTs a bounded continuation when a background run is reaped stale", async () => { vi.useFakeTimers(); const dispatchEvent = vi.fn(); diff --git a/packages/core/src/client/agent-chat-adapter.ts b/packages/core/src/client/agent-chat-adapter.ts index e47e09b7f8..5a40eb3295 100644 --- a/packages/core/src/client/agent-chat-adapter.ts +++ b/packages/core/src/client/agent-chat-adapter.ts @@ -1,6 +1,10 @@ import type { ChatModelAdapter, ChatModelRunResult } from "@assistant-ui/react"; import { actionPreparationContinuationNote } from "../agent/action-continuation-guidance.js"; +import { + LLM_MISSING_CREDENTIALS_ERROR_CODE, + LLM_MISSING_CREDENTIALS_MESSAGE, +} from "../agent/engine/credential-errors.js"; import type { AgentChatStructuredContentPart, AgentChatStructuredMessage, @@ -157,7 +161,9 @@ const BACKGROUND_FOLLOW_IDLE_TIMEOUT_MS = 150_000; * (`agent_runs.terminal_reason`, surfaced by /runs/active). These runs died in * server-side handoff machinery where no richer error event may exist, so the * client owns the message. Keys are matched after stripping an optional - * `error:` prefix (`terminalReasonForEvent` records `error:`). + * `error:` prefix (`terminalReasonForEvent` records `error:`). Keep this + * map scoped to terminal failures, including reasons that must override a + * mistakenly completed row. */ const BACKGROUND_TERMINAL_REASON_MESSAGES: Record = { background_worker_never_started: @@ -166,6 +172,8 @@ const BACKGROUND_TERMINAL_REASON_MESSAGES: Record = { "The agent's background worker could not hand off the next step of this run. Retry to continue from the preserved context.", dispatch_payload_missing: "The agent's background run lost its saved request data and could not continue. Retry to start a fresh run.", + missing_api_key: LLM_MISSING_CREDENTIALS_MESSAGE, + missing_credentials: LLM_MISSING_CREDENTIALS_MESSAGE, turn_continuation_budget_exhausted: "This request needed more automatic continuations than allowed and was stopped. Try breaking it into smaller steps.", }; @@ -1761,6 +1769,11 @@ export function createAgentChatAdapter( nextRunId: string, previousRunId: string | null, ) => { + if (previousRunId !== nextRunId) { + lastAutoContinueReason = null; + lastRecoverableRunError = null; + lastActivityTrail = []; + } const rememberedSeq = seenRunSeqs.get(nextRunId); if (rememberedSeq !== undefined) { lastSeq = rememberedSeq; @@ -2132,6 +2145,9 @@ export function createAgentChatAdapter( ...(runId ? { runId } : {}), }; if (typeof window !== "undefined") { + if (args.errorCode === LLM_MISSING_CREDENTIALS_ERROR_CODE) { + dispatchMissingApiKey(); + } window.dispatchEvent( new CustomEvent("agent-chat:run-error", { detail: { ...runError, tabId }, @@ -2169,7 +2185,46 @@ export function createAgentChatAdapter( : ""; // terminal_reason is either a bare reason ("dispatch_payload_missing") // or "error:" when derived from a terminal error event. + const hasErrorTerminalReason = rawTerminalReason.startsWith("error:"); const terminalReason = rawTerminalReason.replace(/^error:/, ""); + const mappedMessage = terminalReason + ? BACKGROUND_TERMINAL_REASON_MESSAGES[terminalReason] + : undefined; + const mappedErrorCode = + terminalReason === "missing_api_key" || + terminalReason === LLM_MISSING_CREDENTIALS_ERROR_CODE + ? LLM_MISSING_CREDENTIALS_ERROR_CODE + : terminalReason; + + if (mappedMessage) { + yield* emitBackgroundTerminalError({ + message: mappedMessage, + errorCode: mappedErrorCode, + details: `terminal_reason: ${rawTerminalReason}`, + }); + return; + } + + if (hasErrorTerminalReason) { + if (lastRecoverableRunError) { + yield* emitBackgroundTerminalError({ + message: lastRecoverableRunError.message, + errorCode: + lastRecoverableRunError.errorCode || + mappedErrorCode || + "background_run_failed", + details: lastRecoverableRunError.details, + }); + return; + } + yield* emitBackgroundTerminalError({ + message: + "The agent's background run failed before its final response could be recovered. You can retry from the preserved chat context.", + errorCode: mappedErrorCode || "background_run_failed", + details: `terminal_reason: ${rawTerminalReason}`, + }); + return; + } if (status === "completed") { // The turn finished server-side; the events we managed to fold are @@ -2186,17 +2241,6 @@ export function createAgentChatAdapter( return; } - const mappedMessage = terminalReason - ? BACKGROUND_TERMINAL_REASON_MESSAGES[terminalReason] - : undefined; - if (mappedMessage) { - yield* emitBackgroundTerminalError({ - message: mappedMessage, - errorCode: terminalReason, - details: `terminal_reason: ${rawTerminalReason}`, - }); - return; - } if (lastRecoverableRunError) { // A replayed terminal error event already carried the real // message (recoverable errors replay as auto-continue signals