From aabfff90cb3d643c92e9a2d6730247f4fb5874a7 Mon Sep 17 00:00:00 2001 From: Ho Lim Date: Wed, 8 Jul 2026 18:18:42 +0000 Subject: [PATCH 01/12] fix(ollama): warm unloaded model after daemon restart Signed-off-by: Ho Lim Signed-off-by: cjagwani --- docs/inference/use-local-inference.mdx | 2 + docs/reference/troubleshooting.mdx | 2 + .../agent/ollama-restart-recovery.test.ts | 210 +++++++++++++++++ .../sandbox/agent/ollama-restart-recovery.ts | 222 ++++++++++++++++++ .../actions/sandbox/agent/passthrough.test.ts | 99 +++++++- src/lib/actions/sandbox/agent/passthrough.ts | 94 +++++++- 6 files changed, 626 insertions(+), 3 deletions(-) create mode 100644 src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts create mode 100644 src/lib/actions/sandbox/agent/ollama-restart-recovery.ts diff --git a/docs/inference/use-local-inference.mdx b/docs/inference/use-local-inference.mdx index cb366d42d9..8d862c8be8 100644 --- a/docs/inference/use-local-inference.mdx +++ b/docs/inference/use-local-inference.mdx @@ -92,6 +92,8 @@ If the selected model declares that it does not support tool calling, onboarding The validation also requires structured chat-completions tool calls. If the model leaks tool-call JSON as plain message text, onboarding stops so you can choose a model that returns tool calls in the expected response field. If a host-side validation probe times out, NemoClaw retries the Ollama tool-call validation with a larger timeout before failing the setup. +After an Ollama daemon restart, the first agent passthrough checks the registered route to see whether the selected model is still loaded. +If the daemon is reachable but the model is unloaded, NemoClaw sends a bounded warm-up request before dispatching to OpenClaw. On WSL, if you choose the Windows-host Ollama path, NemoClaw uses `host.docker.internal:11434` and pulls missing models through the Ollama HTTP API instead of requiring the `ollama` CLI inside WSL. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 9080cb3950..0bf8295b0c 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1342,6 +1342,8 @@ For local Ollama and local vLLM, `$$nemoclaw status` also prints an `Infe If that line shows `unreachable`, start the local backend first and then retry the request. For Local Ollama, current releases also print `Inference (auth proxy)` when a proxy token is available. If the backend is healthy but the auth proxy is `unauthorized` or `unreachable`, re-run onboarding so NemoClaw can recreate the proxy token, restart the proxy, and refresh the route. +For Ollama-backed OpenClaw sandboxes, agent passthrough uses the registered host route to warm an unloaded model after an Ollama daemon restart. +If that bounded warm-up fails or times out, NemoClaw reports the result and continues so OpenClaw can emit its canonical backend error. If the endpoint is correct but requests still fail, check for network policy rules that may block the connection. Then verify the credential and base URL for the provider you selected during onboarding. diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts new file mode 100644 index 0000000000..f79557242f --- /dev/null +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -0,0 +1,210 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; +import { + maybeWarmOllamaAfterDaemonRestart, + type OllamaRestartRecoveryDeps, +} from "./ollama-restart-recovery"; + +const unloadedStatus = { + probed: true, + loaded: false, + cpuOnly: false, +}; + +function successfulWarmResult() { + return { + stdout: JSON.stringify({ response: "Hello!", done: true }), + exitCode: 0, + timedOut: false, + }; +} + +function getCommandUrl(command: readonly string[]): string { + return command.find((arg) => arg.startsWith("http://")) ?? ""; +} + +describe("maybeWarmOllamaAfterDaemonRestart", () => { + it("skips routes that are not local Ollama", () => { + expect( + maybeWarmOllamaAfterDaemonRestart({ provider: "vllm-local", model: "meta/llama" }), + ).toEqual({ kind: "skipped", reason: "not-ollama" }); + }); + + it("skips a local Ollama route without a registered model", () => { + expect(maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local" })).toEqual({ + kind: "skipped", + reason: "missing-model", + }); + }); + + it("uses the persisted direct bridge route for both the default probe and warm-up", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, + }, + { runCaptureImpl, runCaptureExImpl }, + ), + ).toEqual({ kind: "warmed", ok: true, timedOut: false }); + + expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, + ); + expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, + ); + }); + + it("maps an auth-proxy route back to host loopback", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://host.openshell.internal:${OLLAMA_PROXY_PORT}/v1`, + }, + { runCaptureImpl, runCaptureExImpl }, + ); + + expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + `http://127.0.0.1:${OLLAMA_PORT}/api/ps`, + ); + expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + `http://127.0.0.1:${OLLAMA_PORT}/api/generate`, + ); + }); + + it("falls back to an allowlisted host instead of probing an arbitrary registry URL", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://example.com:${OLLAMA_PORT}/v1`, + }, + { + getOllamaHost: () => "also.example.com", + runCaptureImpl, + runCaptureExImpl, + }, + ); + + expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); + expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); + }); + + it("skips the warm-up when the selected model is already loaded", () => { + const probeRuntimeModelStatus = vi.fn(() => ({ + probed: true, + loaded: true, + cpuOnly: false, + })); + const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { probeRuntimeModelStatus, runCaptureExImpl }, + ), + ).toEqual({ kind: "skipped", reason: "already-loaded" }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("skips the warm-up when the daemon probe is unreachable", () => { + const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { runCaptureImpl: () => "", runCaptureExImpl }, + ), + ).toEqual({ kind: "skipped", reason: "unreachable" }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("reports a bounded warm-up timeout", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ + stdout: "", + exitCode: 28, + timedOut: true, + }), + }, + ), + ).toEqual({ kind: "warmed", ok: false, timedOut: true, reason: "timeout" }); + }); + + it("does not treat an exit-zero Ollama error body as a successful warm-up", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "missing:latest" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ + stdout: JSON.stringify({ error: "model not found" }), + exitCode: 0, + timedOut: false, + }), + }, + ), + ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "ollama-error" }); + }); + + it.each([ + ["empty body", ""], + ["malformed JSON", "not-json"], + ["missing done marker", JSON.stringify({ response: "Hello!" })], + ["empty response", JSON.stringify({ response: "", done: true })], + ])("rejects an invalid warm response: %s", (_name, stdout) => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ stdout, exitCode: 0, timedOut: false }), + }, + ), + ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "invalid-response" }); + }); + + it("reports a non-zero warm command exit", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ stdout: "", exitCode: 7, timedOut: false }), + }, + ), + ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "command-failed" }); + }); + + it("reports a warm process spawn failure without throwing", () => { + const deps: OllamaRestartRecoveryDeps = { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => { + throw new Error("spawn failed"); + }, + }; + + expect( + maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local", model: "qwen3.6:35b" }, deps), + ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "spawn-failed" }); + }); +}); diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts new file mode 100644 index 0000000000..3f6479c8e1 --- /dev/null +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args"; +import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; +import { + getResolvedOllamaHost, + OLLAMA_HOST_DOCKER_INTERNAL, + OLLAMA_LOCALHOST, + type RunCaptureExFn, +} from "../../../inference/local"; +import { + type OllamaRuntimeModelStatus, + type OllamaRuntimeRunCaptureFn, + probeOllamaRuntimeModelStatus, +} from "../../../inference/ollama-runtime-context"; +import { runCaptureEx } from "../../../runner"; + +export interface OllamaRestartRecoveryRoute { + provider?: string | null; + model?: string | null; + endpointUrl?: string | null; +} + +export interface OllamaRestartRecoveryDeps { + probeRuntimeModelStatus?: ( + model: string, + getOllamaHost: () => string, + runCaptureImpl?: OllamaRuntimeRunCaptureFn, + ) => OllamaRuntimeModelStatus; + runCaptureExImpl?: RunCaptureExFn; + getOllamaHost?: () => string; + runCaptureImpl?: OllamaRuntimeRunCaptureFn; +} + +export type OllamaRestartRecoveryFailureReason = + | "timeout" + | "command-failed" + | "ollama-error" + | "invalid-response" + | "spawn-failed"; + +export type OllamaRestartRecoveryResult = + | { kind: "skipped"; reason: "not-ollama" | "missing-model" | "already-loaded" | "unreachable" } + | { kind: "warmed"; ok: true; timedOut: false } + | { + kind: "warmed"; + ok: false; + timedOut: boolean; + reason: OllamaRestartRecoveryFailureReason; + }; + +const OLLAMA_PROVIDER = "ollama-local"; +const OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS = 300; +const OPENSHELL_HOST_BRIDGE = "host.openshell.internal"; +const ALLOWED_RAW_OLLAMA_HOSTS = new Set([ + OLLAMA_LOCALHOST, + "localhost", + "::1", + OLLAMA_HOST_DOCKER_INTERNAL, +]); + +function normalizeRouteValue(value: string | null | undefined): string { + return String(value ?? "").trim(); +} + +function normalizeHostname(value: string): string { + return (value.startsWith("[") && value.endsWith("]") ? value.slice(1, -1) : value) + .replace(/\.$/, "") + .toLowerCase(); +} + +function getAllowedFallbackHost(getOllamaHost: () => string): string { + try { + const host = normalizeHostname(getOllamaHost()); + return ALLOWED_RAW_OLLAMA_HOSTS.has(host) ? host : OLLAMA_LOCALHOST; + } catch { + return OLLAMA_LOCALHOST; + } +} + +/** + * Translate the persisted sandbox-facing route back to the host-side daemon. + * Only fixed local bridge names are accepted so edited registry data cannot + * turn this recovery probe into an arbitrary host request. + */ +function resolveRawOllamaHost( + endpointUrl: string | null | undefined, + getOllamaHost: () => string, +): string { + try { + const endpoint = new URL(normalizeRouteValue(endpointUrl)); + const hostname = normalizeHostname(endpoint.hostname); + const port = Number(endpoint.port || (endpoint.protocol === "https:" ? 443 : 80)); + + if ( + endpoint.protocol === "http:" && + hostname === OPENSHELL_HOST_BRIDGE && + port === OLLAMA_PORT + ) { + return OLLAMA_HOST_DOCKER_INTERNAL; + } + if (endpoint.protocol === "http:" && port === OLLAMA_PROXY_PORT) { + return OLLAMA_LOCALHOST; + } + if ( + endpoint.protocol === "http:" && + port === OLLAMA_PORT && + ALLOWED_RAW_OLLAMA_HOSTS.has(hostname) + ) { + return hostname; + } + } catch { + // Missing and legacy registry endpoints use the process-local resolved host. + } + + return getAllowedFallbackHost(getOllamaHost); +} + +function formatUrlHostname(hostname: string): string { + return hostname.includes(":") ? `[${hostname}]` : hostname; +} + +function buildWarmCommand(model: string, hostname: string): string[] { + const body = JSON.stringify({ + model, + prompt: "Hello, reply in less than 5 words", + stream: false, + keep_alive: "15m", + options: { num_predict: 16 }, + }); + return [ + "curl", + ...buildValidatedCurlCommandArgs([ + "-sS", + "--connect-timeout", + "3", + "--max-time", + String(OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS), + "-H", + "Content-Type: application/json", + "-d", + body, + `http://${formatUrlHostname(hostname)}:${OLLAMA_PORT}/api/generate`, + ]), + ]; +} + +function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid-response" { + try { + const parsed = JSON.parse(stdout) as { + done?: unknown; + error?: unknown; + response?: unknown; + }; + if (typeof parsed.error === "string" && parsed.error.trim() !== "") { + return "ollama-error"; + } + if ( + parsed.done !== true || + typeof parsed.response !== "string" || + parsed.response.trim() === "" + ) { + return "invalid-response"; + } + return "ok"; + } catch { + return "invalid-response"; + } +} + +/** + * Warm a registered local Ollama model only when `/api/ps` proves that the + * daemon is reachable and the selected model is no longer loaded. + */ +export function maybeWarmOllamaAfterDaemonRestart( + route: OllamaRestartRecoveryRoute, + deps: OllamaRestartRecoveryDeps = {}, +): OllamaRestartRecoveryResult { + if (normalizeRouteValue(route.provider) !== OLLAMA_PROVIDER) { + return { kind: "skipped", reason: "not-ollama" }; + } + + const model = normalizeRouteValue(route.model); + if (!model) { + return { kind: "skipped", reason: "missing-model" }; + } + + const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost; + const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); + const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; + let status: OllamaRuntimeModelStatus; + try { + status = probe(model, () => rawHost, deps.runCaptureImpl); + } catch { + return { kind: "skipped", reason: "unreachable" }; + } + if (!status.probed) { + return { kind: "skipped", reason: "unreachable" }; + } + if (status.loaded) { + return { kind: "skipped", reason: "already-loaded" }; + } + + const captureEx = deps.runCaptureExImpl ?? runCaptureEx; + try { + const result = captureEx(buildWarmCommand(model, rawHost)); + if (result.timedOut) { + return { kind: "warmed", ok: false, timedOut: true, reason: "timeout" }; + } + if (result.exitCode !== 0) { + return { kind: "warmed", ok: false, timedOut: false, reason: "command-failed" }; + } + const response = validateWarmResponse(result.stdout); + if (response !== "ok") { + return { kind: "warmed", ok: false, timedOut: false, reason: response }; + } + return { kind: "warmed", ok: true, timedOut: false }; + } catch { + return { kind: "warmed", ok: false, timedOut: false, reason: "spawn-failed" }; + } +} diff --git a/src/lib/actions/sandbox/agent/passthrough.test.ts b/src/lib/actions/sandbox/agent/passthrough.test.ts index de1bf2d267..ad51b38b0a 100644 --- a/src/lib/actions/sandbox/agent/passthrough.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough.test.ts @@ -7,7 +7,17 @@ const execMock = vi.hoisted(() => vi.fn(async () => {})); const ensureLiveMock = vi.hoisted(() => vi.fn(async () => ({ state: "present", output: "Phase: Ready" }) as { output?: string }), ); -const getSandboxMock = vi.hoisted(() => vi.fn(() => null as { agent?: string | null } | null)); +const getSandboxMock = vi.hoisted(() => + vi.fn( + () => + null as { + agent?: string | null; + provider?: string | null; + model?: string | null; + endpointUrl?: string | null; + } | null, + ), +); const listAgentsMock = vi.hoisted(() => vi.fn(() => ["custom-terminal", "hermes", "langchain-deepagents-code", "openclaw"]), ); @@ -112,6 +122,93 @@ describe("runAgentPassthrough", () => { ); }); + it("checks the persisted Ollama route before OpenClaw JSON dispatch", async () => { + const execJson = vi.fn(() => { + throw new Error("__exit:0"); + }); + const maybeWarmOllamaAfterDaemonRestart = vi.fn(() => ({ + kind: "skipped" as const, + reason: "already-loaded" as const, + })); + getSandboxMock.mockReturnValueOnce({ + agent: "openclaw", + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }); + const { writes, proc } = makeProcMock(); + + await expect( + runAgentPassthrough( + "alpha", + { + extraArgs: ["--agent", "work", "--session-id", "s-1", "-m", "ping", "--json"], + }, + { execJson, maybeWarmOllamaAfterDaemonRestart, process: proc }, + ), + ).rejects.toThrow("__exit:0"); + + expect(maybeWarmOllamaAfterDaemonRestart).toHaveBeenCalledWith({ + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }); + expect(maybeWarmOllamaAfterDaemonRestart.mock.invocationCallOrder[0]).toBeLessThan( + execJson.mock.invocationCallOrder[0], + ); + expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is already loaded"); + }); + + it("reports a timed-out Ollama warm-up and still dispatches to OpenClaw", async () => { + const maybeWarmOllamaAfterDaemonRestart = vi.fn(() => ({ + kind: "warmed" as const, + ok: false as const, + timedOut: true, + reason: "timeout" as const, + })); + getSandboxMock.mockReturnValueOnce({ + agent: "openclaw", + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }); + const { writes, proc } = makeProcMock(); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "work", "-m", "ping"] }, + { maybeWarmOllamaAfterDaemonRestart, process: proc }, + ); + + const stderr = writes.join(""); + expect(stderr).toContain("Checking Ollama model readiness after daemon restart"); + expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b' timed out"); + expect(stderr).toContain("continuing to OpenClaw dispatch"); + expect(maybeWarmOllamaAfterDaemonRestart.mock.invocationCallOrder[0]).toBeLessThan( + execMock.mock.invocationCallOrder[0], + ); + expect(execMock).toHaveBeenCalledOnce(); + }); + + it("does not run Ollama restart recovery for a non-Ollama route", async () => { + const maybeWarmOllamaAfterDaemonRestart = vi.fn(); + getSandboxMock.mockReturnValueOnce({ + agent: "openclaw", + provider: "vllm-local", + model: "meta/llama", + endpointUrl: "http://host.openshell.internal:8000/v1", + }); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "work", "-m", "ping"] }, + { maybeWarmOllamaAfterDaemonRestart }, + ); + + expect(maybeWarmOllamaAfterDaemonRestart).not.toHaveBeenCalled(); + expect(execMock).toHaveBeenCalledOnce(); + }); + it("keeps --json as a message value on the normal passthrough path", async () => { const execJson = vi.fn(((): never => { throw new Error("__unexpected-json"); diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index a9d77d9f96..30780aee52 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -99,6 +99,12 @@ import { parseSandboxPhase } from "../../../state/gateway"; import * as registry from "../../../state/registry"; import { execSandbox } from "../exec"; import { ensureLiveSandboxOrExit } from "../gateway-state"; +import { + maybeWarmOllamaAfterDaemonRestart, + type OllamaRestartRecoveryFailureReason, + type OllamaRestartRecoveryResult, + type OllamaRestartRecoveryRoute, +} from "./ollama-restart-recovery"; import { hasAgentPassthroughHelpToken, printAgentPassthroughHelp } from "./passthrough-help"; import { type AgentJsonPassthroughProcess, runAgentJsonPassthrough } from "./passthrough-json"; import { maybeEmitShieldsRelockWarning } from "./passthrough-shields-warning"; @@ -134,6 +140,9 @@ export interface AgentPassthroughDeps { ensureLive?: typeof ensureLiveSandboxOrExit; exec?: typeof execSandbox; execJson?: typeof runAgentJsonPassthrough; + maybeWarmOllamaAfterDaemonRestart?: ( + route: OllamaRestartRecoveryRoute, + ) => OllamaRestartRecoveryResult; getRecentShieldsAutoRestore?: (sandboxName: string) => ShieldsAutoRestoreReadResult; process?: { exit(code: number): never; @@ -144,7 +153,13 @@ export interface AgentPassthroughDeps { type RegistryReadResult = | { kind: "missing" } - | { kind: "agent"; agent: string | null } + | { + kind: "agent"; + agent: string | null; + provider: string | null; + model: string | null; + endpointUrl: string | null; + } | { kind: "error"; message: string }; type ResolvedRegistryReadResult = Exclude; type TerminalCommandResult = @@ -158,12 +173,74 @@ function readSandboxAgentFromRegistry( try { const sandbox = getSandbox(sandboxName); if (!sandbox) return { kind: "missing" }; - return { kind: "agent", agent: sandbox.agent ?? null }; + return { + kind: "agent", + agent: sandbox.agent ?? null, + provider: sandbox.provider ?? null, + model: sandbox.model ?? null, + endpointUrl: sandbox.endpointUrl ?? null, + }; } catch (error) { return { kind: "error", message: (error as Error).message ?? String(error) }; } } +function getOllamaRestartRecoveryRoute( + lookup: ResolvedRegistryReadResult, +): OllamaRestartRecoveryRoute | null { + if (lookup.kind !== "agent" || lookup.provider !== "ollama-local") return null; + return { + provider: lookup.provider, + model: lookup.model, + endpointUrl: lookup.endpointUrl, + }; +} + +function describeOllamaWarmFailure(reason: OllamaRestartRecoveryFailureReason): string { + switch (reason) { + case "timeout": + return "timed out"; + case "command-failed": + return "curl exited unsuccessfully"; + case "ollama-error": + return "Ollama returned an error"; + case "invalid-response": + return "Ollama returned an invalid response"; + case "spawn-failed": + return "the warm-up process could not start"; + } +} + +function reportOllamaRestartRecovery( + route: OllamaRestartRecoveryRoute, + result: OllamaRestartRecoveryResult, + proc: NonNullable, +): void { + const model = String(route.model ?? "").trim() || "the registered model"; + if (result.kind === "warmed") { + if (result.ok) { + proc.stderr.write(` Ollama model '${model}' is loaded and ready.\n`); + return; + } + proc.stderr.write( + ` Ollama warm-up for '${model}' ${describeOllamaWarmFailure(result.reason)}; continuing to OpenClaw dispatch.\n`, + ); + return; + } + + if (result.reason === "already-loaded") { + proc.stderr.write(` Ollama model '${model}' is already loaded.\n`); + } else if (result.reason === "unreachable") { + proc.stderr.write( + " Ollama was unreachable during the restart check; continuing to OpenClaw dispatch.\n", + ); + } else if (result.reason === "missing-model") { + proc.stderr.write( + " No Ollama model is recorded for this sandbox; continuing to OpenClaw dispatch.\n", + ); + } +} + function rejectNonOpenclawAgent( sandboxName: string, agent: string, @@ -425,6 +502,19 @@ export async function runAgentPassthrough( rejectNoTargetSelector(proc); } if (isOpenClawPassthroughCommand(command)) { + const route = getOllamaRestartRecoveryRoute(lookup); + if (route) { + const recoverOllama = + deps.maybeWarmOllamaAfterDaemonRestart ?? maybeWarmOllamaAfterDaemonRestart; + proc.stderr.write(" Checking Ollama model readiness after daemon restart...\n"); + try { + reportOllamaRestartRecovery(route, recoverOllama(route), proc); + } catch { + proc.stderr.write( + " Ollama restart recovery failed unexpectedly; continuing to OpenClaw dispatch.\n", + ); + } + } maybeEmitShieldsRelockWarning(proc, sandboxName, deps.getRecentShieldsAutoRestore); } if (isOpenClawPassthroughCommand(command) && requestsOpenClawJsonOutput(extraArgs)) { From 6a2b44dfba577f2185229384111e599f0ccbb4d2 Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 18:32:53 +0000 Subject: [PATCH 02/12] refactor(ollama): isolate restart recovery orchestration Keep passthrough dispatch focused while preserving route ordering, reporting, and graceful fallback in a dedicated module. Signed-off-by: cjagwani --- .../sandbox/agent/ollama-restart-recovery.ts | 20 +- .../agent/passthrough-ollama-recovery.test.ts | 208 ++++++++++++++++++ .../agent/passthrough-ollama-recovery.ts | 78 +++++++ .../actions/sandbox/agent/passthrough.test.ts | 87 -------- src/lib/actions/sandbox/agent/passthrough.ts | 97 ++------ 5 files changed, 317 insertions(+), 173 deletions(-) create mode 100644 src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts create mode 100644 src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 3f6479c8e1..58a95a8a6c 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -1,6 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// Source-of-truth boundary for Ollama restart recovery. +// +// Invalid state: restarting the external Ollama daemon drops its loaded model, +// so the first OpenClaw turn can exhaust its request budget cold-loading it. +// Ollama owns daemon/model lifecycle; NemoClaw owns the persisted inference +// route and the host-side passthrough that can perform a bounded warm-up before +// dispatch. This cannot be fixed at the producer in this PR because Ollama does +// not persist loaded runners across daemon restarts. Focused tests cover direct +// and proxied route translation, unreachable/already-loaded states, timeouts, +// process failures, and semantic response validation. Remove this recovery when +// supported Ollama versions persist runners across restart, or when NemoClaw +// manages daemon lifecycle and can warm the model at restart time instead. + import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args"; import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; import { @@ -56,7 +69,6 @@ const OPENSHELL_HOST_BRIDGE = "host.openshell.internal"; const ALLOWED_RAW_OLLAMA_HOSTS = new Set([ OLLAMA_LOCALHOST, "localhost", - "::1", OLLAMA_HOST_DOCKER_INTERNAL, ]); @@ -117,10 +129,6 @@ function resolveRawOllamaHost( return getAllowedFallbackHost(getOllamaHost); } -function formatUrlHostname(hostname: string): string { - return hostname.includes(":") ? `[${hostname}]` : hostname; -} - function buildWarmCommand(model: string, hostname: string): string[] { const body = JSON.stringify({ model, @@ -141,7 +149,7 @@ function buildWarmCommand(model: string, hostname: string): string[] { "Content-Type: application/json", "-d", body, - `http://${formatUrlHostname(hostname)}:${OLLAMA_PORT}/api/generate`, + `http://${hostname}:${OLLAMA_PORT}/api/generate`, ]), ]; } diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts new file mode 100644 index 0000000000..696cc9c0b0 --- /dev/null +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { type AgentPassthroughDeps, runAgentPassthrough } from "./passthrough"; +import { runOllamaRestartRecovery } from "./passthrough-ollama-recovery"; + +function makeProcMock() { + const writes: string[] = []; + return { + writes, + proc: { stderr: { write: (value: string) => writes.push(value) } }, + }; +} + +describe("runOllamaRestartRecovery", () => { + it.each([ + ["auth proxy", "http://host.openshell.internal:11435/v1"], + ["WSL direct bridge", "http://host.openshell.internal:11434/v1"], + ])("forwards the persisted %s route to recovery", (_name, endpointUrl) => { + const recoverOllama = vi.fn(() => ({ + kind: "skipped" as const, + reason: "already-loaded" as const, + })); + const { writes, proc } = makeProcMock(); + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl, + }; + + runOllamaRestartRecovery(route, proc, recoverOllama); + + expect(recoverOllama).toHaveBeenCalledWith(route); + expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is already loaded"); + }); + + it("reports a successful warm-up", () => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "warmed", + ok: true, + timedOut: false, + })); + + expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is loaded and ready"); + }); + + it("reports a timeout before continuing to OpenClaw", () => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "warmed", + ok: false, + timedOut: true, + reason: "timeout", + })); + + const stderr = writes.join(""); + expect(stderr).toContain("Checking Ollama model readiness after daemon restart"); + expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b' timed out"); + expect(stderr).toContain("continuing to OpenClaw dispatch"); + }); + + it.each([ + ["command-failed", "curl exited unsuccessfully"], + ["ollama-error", "Ollama returned an error"], + ["invalid-response", "Ollama returned an invalid response"], + ["spawn-failed", "the warm-up process could not start"], + ] as const)("reports a %s warm-up failure", (reason, message) => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "warmed", + ok: false, + timedOut: false, + reason, + })); + + expect(writes.join("")).toContain(message); + }); + + it("reports an unreachable daemon without blocking dispatch", () => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "skipped", + reason: "unreachable", + })); + + expect(writes.join("")).toContain( + "Ollama was unreachable during the restart check; continuing to OpenClaw dispatch", + ); + }); + + it("contains an unexpected recovery exception", () => { + const { writes, proc } = makeProcMock(); + + expect(() => + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => { + throw new Error("unexpected"); + }), + ).not.toThrow(); + expect(writes.join("")).toContain( + "Ollama restart recovery failed unexpectedly; continuing to OpenClaw dispatch", + ); + }); +}); + +function makePassthroughDeps( + route: { provider: string; model: string; endpointUrl: string }, + events: string[], +): AgentPassthroughDeps { + return { + getSandbox: (() => ({ agent: "openclaw", ...route })) as NonNullable< + AgentPassthroughDeps["getSandbox"] + >, + ensureLive: (async () => ({ state: "present", output: "Phase: Ready" })) as NonNullable< + AgentPassthroughDeps["ensureLive"] + >, + exec: (async () => { + events.push("dispatch"); + }) as NonNullable, + getRecentShieldsAutoRestore: () => ({ kind: "none" }), + process: { + exit: ((code: number) => { + throw new Error(`__exit:${code}`); + }) as (code: number) => never, + stdout: { write: () => true }, + stderr: { write: () => true }, + }, + }; +} + +describe("agent passthrough Ollama recovery ordering", () => { + it("checks an auth-proxy route before JSON dispatch", async () => { + const events: string[] = []; + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11435/v1", + }; + const deps = makePassthroughDeps(route, events); + const runRecovery = vi.fn(() => { + events.push("recovery"); + }); + const execJson = vi.fn(((): never => { + events.push("dispatch"); + throw new Error("__exit:0"); + }) as NonNullable); + + await expect( + runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "-m", "ping", "--json"] }, + { ...deps, execJson, runOllamaRestartRecovery: runRecovery }, + ), + ).rejects.toThrow("__exit:0"); + + expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process); + expect(events).toEqual(["recovery", "dispatch"]); + }); + + it("checks a WSL direct route before non-JSON dispatch", async () => { + const events: string[] = []; + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }; + const deps = makePassthroughDeps(route, events); + const runRecovery = vi.fn(() => { + events.push("recovery"); + }); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "-m", "ping"] }, + { ...deps, runOllamaRestartRecovery: runRecovery }, + ); + + expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process); + expect(events).toEqual(["recovery", "dispatch"]); + }); + + it("does not run Ollama recovery for a non-Ollama route", async () => { + const events: string[] = []; + const deps = makePassthroughDeps( + { + provider: "vllm-local", + model: "meta/llama", + endpointUrl: "http://host.openshell.internal:8000/v1", + }, + events, + ); + const runRecovery = vi.fn(); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "-m", "ping"] }, + { ...deps, runOllamaRestartRecovery: runRecovery }, + ); + + expect(runRecovery).not.toHaveBeenCalled(); + expect(events).toEqual(["dispatch"]); + }); +}); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts new file mode 100644 index 0000000000..e06b9a5c73 --- /dev/null +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + maybeWarmOllamaAfterDaemonRestart, + type OllamaRestartRecoveryFailureReason, + type OllamaRestartRecoveryResult, + type OllamaRestartRecoveryRoute, +} from "./ollama-restart-recovery"; + +export type OllamaRestartRecoveryFn = ( + route: OllamaRestartRecoveryRoute, +) => OllamaRestartRecoveryResult; + +export interface OllamaRestartRecoveryProcess { + stderr: { write(s: string): unknown }; +} + +function describeWarmFailure(reason: OllamaRestartRecoveryFailureReason): string { + switch (reason) { + case "timeout": + return "timed out"; + case "command-failed": + return "curl exited unsuccessfully"; + case "ollama-error": + return "Ollama returned an error"; + case "invalid-response": + return "Ollama returned an invalid response"; + case "spawn-failed": + return "the warm-up process could not start"; + } +} + +function reportRecovery( + route: OllamaRestartRecoveryRoute, + result: OllamaRestartRecoveryResult, + proc: OllamaRestartRecoveryProcess, +): void { + const model = String(route.model ?? "").trim() || "the registered model"; + if (result.kind === "warmed") { + if (result.ok) { + proc.stderr.write(` Ollama model '${model}' is loaded and ready.\n`); + return; + } + proc.stderr.write( + ` Ollama warm-up for '${model}' ${describeWarmFailure(result.reason)}; continuing to OpenClaw dispatch.\n`, + ); + return; + } + + if (result.reason === "already-loaded") { + proc.stderr.write(` Ollama model '${model}' is already loaded.\n`); + } else if (result.reason === "unreachable") { + proc.stderr.write( + " Ollama was unreachable during the restart check; continuing to OpenClaw dispatch.\n", + ); + } else if (result.reason === "missing-model") { + proc.stderr.write( + " No Ollama model is recorded for this sandbox; continuing to OpenClaw dispatch.\n", + ); + } +} + +/** Run best-effort Ollama recovery without blocking the canonical agent error path. */ +export function runOllamaRestartRecovery( + route: OllamaRestartRecoveryRoute, + proc: OllamaRestartRecoveryProcess, + recoverOllama: OllamaRestartRecoveryFn = maybeWarmOllamaAfterDaemonRestart, +): void { + proc.stderr.write(" Checking Ollama model readiness after daemon restart...\n"); + try { + reportRecovery(route, recoverOllama(route), proc); + } catch { + proc.stderr.write( + " Ollama restart recovery failed unexpectedly; continuing to OpenClaw dispatch.\n", + ); + } +} diff --git a/src/lib/actions/sandbox/agent/passthrough.test.ts b/src/lib/actions/sandbox/agent/passthrough.test.ts index ad51b38b0a..a76f2e0ad6 100644 --- a/src/lib/actions/sandbox/agent/passthrough.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough.test.ts @@ -122,93 +122,6 @@ describe("runAgentPassthrough", () => { ); }); - it("checks the persisted Ollama route before OpenClaw JSON dispatch", async () => { - const execJson = vi.fn(() => { - throw new Error("__exit:0"); - }); - const maybeWarmOllamaAfterDaemonRestart = vi.fn(() => ({ - kind: "skipped" as const, - reason: "already-loaded" as const, - })); - getSandboxMock.mockReturnValueOnce({ - agent: "openclaw", - provider: "ollama-local", - model: "qwen3.6:35b", - endpointUrl: "http://host.openshell.internal:11434/v1", - }); - const { writes, proc } = makeProcMock(); - - await expect( - runAgentPassthrough( - "alpha", - { - extraArgs: ["--agent", "work", "--session-id", "s-1", "-m", "ping", "--json"], - }, - { execJson, maybeWarmOllamaAfterDaemonRestart, process: proc }, - ), - ).rejects.toThrow("__exit:0"); - - expect(maybeWarmOllamaAfterDaemonRestart).toHaveBeenCalledWith({ - provider: "ollama-local", - model: "qwen3.6:35b", - endpointUrl: "http://host.openshell.internal:11434/v1", - }); - expect(maybeWarmOllamaAfterDaemonRestart.mock.invocationCallOrder[0]).toBeLessThan( - execJson.mock.invocationCallOrder[0], - ); - expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is already loaded"); - }); - - it("reports a timed-out Ollama warm-up and still dispatches to OpenClaw", async () => { - const maybeWarmOllamaAfterDaemonRestart = vi.fn(() => ({ - kind: "warmed" as const, - ok: false as const, - timedOut: true, - reason: "timeout" as const, - })); - getSandboxMock.mockReturnValueOnce({ - agent: "openclaw", - provider: "ollama-local", - model: "qwen3.6:35b", - endpointUrl: "http://host.openshell.internal:11434/v1", - }); - const { writes, proc } = makeProcMock(); - - await runAgentPassthrough( - "alpha", - { extraArgs: ["--agent", "work", "-m", "ping"] }, - { maybeWarmOllamaAfterDaemonRestart, process: proc }, - ); - - const stderr = writes.join(""); - expect(stderr).toContain("Checking Ollama model readiness after daemon restart"); - expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b' timed out"); - expect(stderr).toContain("continuing to OpenClaw dispatch"); - expect(maybeWarmOllamaAfterDaemonRestart.mock.invocationCallOrder[0]).toBeLessThan( - execMock.mock.invocationCallOrder[0], - ); - expect(execMock).toHaveBeenCalledOnce(); - }); - - it("does not run Ollama restart recovery for a non-Ollama route", async () => { - const maybeWarmOllamaAfterDaemonRestart = vi.fn(); - getSandboxMock.mockReturnValueOnce({ - agent: "openclaw", - provider: "vllm-local", - model: "meta/llama", - endpointUrl: "http://host.openshell.internal:8000/v1", - }); - - await runAgentPassthrough( - "alpha", - { extraArgs: ["--agent", "work", "-m", "ping"] }, - { maybeWarmOllamaAfterDaemonRestart }, - ); - - expect(maybeWarmOllamaAfterDaemonRestart).not.toHaveBeenCalled(); - expect(execMock).toHaveBeenCalledOnce(); - }); - it("keeps --json as a message value on the normal passthrough path", async () => { const execJson = vi.fn(((): never => { throw new Error("__unexpected-json"); diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index 30780aee52..2c1c2d29ea 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -3,8 +3,8 @@ // Source-of-truth boundary for the `nemoclaw agent` passthrough. // -// The wrapper enforces three host-side mirrors of upstream contracts and one -// advisory diagnostic: +// The wrapper enforces three host-side mirrors of upstream contracts, one +// advisory diagnostic, and one best-effort pre-dispatch recovery: // // 1. Agent-kind guard (registry mirror). // @@ -72,6 +72,12 @@ // source-boundary analysis lives with the focused implementation in // `passthrough-shields-warning.ts`. // +// 5. Ollama restart recovery (best-effort lifecycle bridge). Ollama owns model +// runner lifetime, while NemoClaw owns the registered route and dispatch +// ordering. The focused source-boundary analysis, reporting, and regression +// coverage live in `ollama-restart-recovery.ts` and +// `passthrough-ollama-recovery.ts`. +// // Regression tests: `passthrough.test.ts` covers the Hermes redirect, the // forwarded argv, the registry-miss fallback to OpenClaw, registry and // manifest-resolution fail-closed paths, quoted manifest command rejection, @@ -79,7 +85,8 @@ // unparseable phase fail-closed path, the OpenClaw no-selector rejection, and // the `--flag=value` selector-acceptance branch, plus the OpenClaw JSON // captured transport path used to append failure provenance without polluting -// machine-readable stdout. The focused shields diagnostic owns its tests. +// machine-readable stdout. The focused shields and Ollama modules own their +// diagnostic and recovery tests. // // Removal conditions: // @@ -91,6 +98,8 @@ // missing selector with a clean exit 2 and an actionable message. // - Drop the simple-token parser when terminal runtime manifests expose // argv arrays natively. +// - Drop Ollama pre-dispatch recovery when supported daemon restarts preserve +// loaded runners or NemoClaw manages and warms the daemon lifecycle. import { type AgentDefinition, isTerminalAgent, listAgents, loadAgent } from "../../../agent/defs"; import { CLI_NAME } from "../../../cli/branding"; @@ -99,14 +108,9 @@ import { parseSandboxPhase } from "../../../state/gateway"; import * as registry from "../../../state/registry"; import { execSandbox } from "../exec"; import { ensureLiveSandboxOrExit } from "../gateway-state"; -import { - maybeWarmOllamaAfterDaemonRestart, - type OllamaRestartRecoveryFailureReason, - type OllamaRestartRecoveryResult, - type OllamaRestartRecoveryRoute, -} from "./ollama-restart-recovery"; import { hasAgentPassthroughHelpToken, printAgentPassthroughHelp } from "./passthrough-help"; import { type AgentJsonPassthroughProcess, runAgentJsonPassthrough } from "./passthrough-json"; +import { runOllamaRestartRecovery } from "./passthrough-ollama-recovery"; import { maybeEmitShieldsRelockWarning } from "./passthrough-shields-warning"; export { @@ -140,9 +144,7 @@ export interface AgentPassthroughDeps { ensureLive?: typeof ensureLiveSandboxOrExit; exec?: typeof execSandbox; execJson?: typeof runAgentJsonPassthrough; - maybeWarmOllamaAfterDaemonRestart?: ( - route: OllamaRestartRecoveryRoute, - ) => OllamaRestartRecoveryResult; + runOllamaRestartRecovery?: typeof runOllamaRestartRecovery; getRecentShieldsAutoRestore?: (sandboxName: string) => ShieldsAutoRestoreReadResult; process?: { exit(code: number): never; @@ -185,62 +187,6 @@ function readSandboxAgentFromRegistry( } } -function getOllamaRestartRecoveryRoute( - lookup: ResolvedRegistryReadResult, -): OllamaRestartRecoveryRoute | null { - if (lookup.kind !== "agent" || lookup.provider !== "ollama-local") return null; - return { - provider: lookup.provider, - model: lookup.model, - endpointUrl: lookup.endpointUrl, - }; -} - -function describeOllamaWarmFailure(reason: OllamaRestartRecoveryFailureReason): string { - switch (reason) { - case "timeout": - return "timed out"; - case "command-failed": - return "curl exited unsuccessfully"; - case "ollama-error": - return "Ollama returned an error"; - case "invalid-response": - return "Ollama returned an invalid response"; - case "spawn-failed": - return "the warm-up process could not start"; - } -} - -function reportOllamaRestartRecovery( - route: OllamaRestartRecoveryRoute, - result: OllamaRestartRecoveryResult, - proc: NonNullable, -): void { - const model = String(route.model ?? "").trim() || "the registered model"; - if (result.kind === "warmed") { - if (result.ok) { - proc.stderr.write(` Ollama model '${model}' is loaded and ready.\n`); - return; - } - proc.stderr.write( - ` Ollama warm-up for '${model}' ${describeOllamaWarmFailure(result.reason)}; continuing to OpenClaw dispatch.\n`, - ); - return; - } - - if (result.reason === "already-loaded") { - proc.stderr.write(` Ollama model '${model}' is already loaded.\n`); - } else if (result.reason === "unreachable") { - proc.stderr.write( - " Ollama was unreachable during the restart check; continuing to OpenClaw dispatch.\n", - ); - } else if (result.reason === "missing-model") { - proc.stderr.write( - " No Ollama model is recorded for this sandbox; continuing to OpenClaw dispatch.\n", - ); - } -} - function rejectNonOpenclawAgent( sandboxName: string, agent: string, @@ -502,18 +448,9 @@ export async function runAgentPassthrough( rejectNoTargetSelector(proc); } if (isOpenClawPassthroughCommand(command)) { - const route = getOllamaRestartRecoveryRoute(lookup); - if (route) { - const recoverOllama = - deps.maybeWarmOllamaAfterDaemonRestart ?? maybeWarmOllamaAfterDaemonRestart; - proc.stderr.write(" Checking Ollama model readiness after daemon restart...\n"); - try { - reportOllamaRestartRecovery(route, recoverOllama(route), proc); - } catch { - proc.stderr.write( - " Ollama restart recovery failed unexpectedly; continuing to OpenClaw dispatch.\n", - ); - } + if (lookup.kind === "agent" && lookup.provider === "ollama-local") { + const recoverOllama = deps.runOllamaRestartRecovery ?? runOllamaRestartRecovery; + recoverOllama(lookup, proc); } maybeEmitShieldsRelockWarning(proc, sandboxName, deps.getRecentShieldsAutoRestore); } From 87e9ed76b9dada0f5e2d8086b433db860c9b8d42 Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 18:50:45 +0000 Subject: [PATCH 03/12] test(ollama): return complete sandbox fixture Signed-off-by: cjagwani --- .../actions/sandbox/agent/passthrough-ollama-recovery.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 696cc9c0b0..2c3415f6ed 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -113,7 +113,7 @@ function makePassthroughDeps( events: string[], ): AgentPassthroughDeps { return { - getSandbox: (() => ({ agent: "openclaw", ...route })) as NonNullable< + getSandbox: ((name) => ({ name, agent: "openclaw", ...route })) as NonNullable< AgentPassthroughDeps["getSandbox"] >, ensureLive: (async () => ({ state: "present", output: "Phase: Ready" })) as NonNullable< From b23e769b60cc6ee421f0b36ffba478dfc8c4876f Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 19:01:46 +0000 Subject: [PATCH 04/12] fix(ollama): restrict persisted proxy route host Signed-off-by: cjagwani --- .../agent/ollama-restart-recovery.test.ts | 25 +++++++++++++++++++ .../sandbox/agent/ollama-restart-recovery.ts | 6 ++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index f79557242f..ff42c2f16e 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -105,6 +105,31 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); }); + it("does not map an unrecognized proxy-port host to host loopback (#6039)", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://example.com:${OLLAMA_PROXY_PORT}/v1`, + }, + { + getOllamaHost: () => "host.docker.internal", + runCaptureImpl, + runCaptureExImpl, + }, + ); + + expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, + ); + expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, + ); + }); + it("skips the warm-up when the selected model is already loaded", () => { const probeRuntimeModelStatus = vi.fn(() => ({ probed: true, diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 58a95a8a6c..52dd7e52a4 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -112,7 +112,11 @@ function resolveRawOllamaHost( ) { return OLLAMA_HOST_DOCKER_INTERNAL; } - if (endpoint.protocol === "http:" && port === OLLAMA_PROXY_PORT) { + if ( + endpoint.protocol === "http:" && + hostname === OPENSHELL_HOST_BRIDGE && + port === OLLAMA_PROXY_PORT + ) { return OLLAMA_LOCALHOST; } if ( From 2b75e25540273997036c163af90d216b9c5a6ce4 Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 20:01:25 +0000 Subject: [PATCH 05/12] refactor(inference): share Ollama provider identity Signed-off-by: cjagwani --- src/lib/actions/sandbox/agent/ollama-restart-recovery.ts | 4 ++-- src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts | 3 +++ src/lib/actions/sandbox/agent/passthrough.ts | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index 52dd7e52a4..b6017f4c3d 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -63,7 +63,7 @@ export type OllamaRestartRecoveryResult = reason: OllamaRestartRecoveryFailureReason; }; -const OLLAMA_PROVIDER = "ollama-local"; +export const OLLAMA_LOCAL_PROVIDER = "ollama-local"; const OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS = 300; const OPENSHELL_HOST_BRIDGE = "host.openshell.internal"; const ALLOWED_RAW_OLLAMA_HOSTS = new Set([ @@ -189,7 +189,7 @@ export function maybeWarmOllamaAfterDaemonRestart( route: OllamaRestartRecoveryRoute, deps: OllamaRestartRecoveryDeps = {}, ): OllamaRestartRecoveryResult { - if (normalizeRouteValue(route.provider) !== OLLAMA_PROVIDER) { + if (normalizeRouteValue(route.provider) !== OLLAMA_LOCAL_PROVIDER) { return { kind: "skipped", reason: "not-ollama" }; } diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index e06b9a5c73..41044fdab9 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -3,11 +3,14 @@ import { maybeWarmOllamaAfterDaemonRestart, + OLLAMA_LOCAL_PROVIDER, type OllamaRestartRecoveryFailureReason, type OllamaRestartRecoveryResult, type OllamaRestartRecoveryRoute, } from "./ollama-restart-recovery"; +export { OLLAMA_LOCAL_PROVIDER }; + export type OllamaRestartRecoveryFn = ( route: OllamaRestartRecoveryRoute, ) => OllamaRestartRecoveryResult; diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index 2c1c2d29ea..096ae9096e 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -110,7 +110,7 @@ import { execSandbox } from "../exec"; import { ensureLiveSandboxOrExit } from "../gateway-state"; import { hasAgentPassthroughHelpToken, printAgentPassthroughHelp } from "./passthrough-help"; import { type AgentJsonPassthroughProcess, runAgentJsonPassthrough } from "./passthrough-json"; -import { runOllamaRestartRecovery } from "./passthrough-ollama-recovery"; +import { OLLAMA_LOCAL_PROVIDER, runOllamaRestartRecovery } from "./passthrough-ollama-recovery"; import { maybeEmitShieldsRelockWarning } from "./passthrough-shields-warning"; export { @@ -448,7 +448,7 @@ export async function runAgentPassthrough( rejectNoTargetSelector(proc); } if (isOpenClawPassthroughCommand(command)) { - if (lookup.kind === "agent" && lookup.provider === "ollama-local") { + if (lookup.kind === "agent" && lookup.provider === OLLAMA_LOCAL_PROVIDER) { const recoverOllama = deps.runOllamaRestartRecovery ?? runOllamaRestartRecovery; recoverOllama(lookup, proc); } From a6b03c56fb7e4b713057d7edc1a37fd1df8beb9d Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 20:13:15 +0000 Subject: [PATCH 06/12] test(e2e): prove Ollama restart recovery Signed-off-by: cjagwani --- test/e2e/live/gpu-e2e.test.ts | 71 +++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index dc230edacb..85351d602f 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -89,6 +89,14 @@ function assertSmallContextCompactionPolicy(configText: string): void { }); } +function loadedOllamaModels(raw: string): string[] { + const parsed = JSON.parse(raw) as { models?: Array<{ name?: unknown; model?: unknown }> }; + return (parsed.models ?? []).flatMap((entry) => { + const name = typeof entry.name === "string" ? entry.name : entry.model; + return typeof name === "string" && name.trim() ? [name.trim()] : []; + }); +} + test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { timeout: TIMEOUT_MS, }, async ({ artifacts, cleanup, host, sandbox, skip }) => { @@ -222,4 +230,67 @@ test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { ); expect(chat.exitCode, resultText(chat)).toBe(0); expect(chatContent(chat.stdout)).toMatch(/pong/i); + + const restart = await host.command( + "bash", + [ + "-lc", + `set -euo pipefail +if sudo -n systemctl restart ollama 2>/dev/null; then + restart_mode=system +elif systemctl --user restart ollama 2>/dev/null; then + restart_mode=user +else + pkill -f '[o]llama serve' 2>/dev/null || true + OLLAMA_HOST=127.0.0.1:11434 nohup ollama serve >/tmp/nemoclaw-gpu-e2e-ollama.log 2>&1 & + restart_mode=manual +fi +for attempt in $(seq 1 60); do + if curl -fsS --connect-timeout 2 http://127.0.0.1:11434/api/tags >/dev/null; then + echo "restart_mode=$restart_mode" + curl -fsS http://127.0.0.1:11434/api/ps + exit 0 + fi + sleep 1 +done +echo 'Ollama did not become ready after restart' >&2 +exit 1`, + ], + { artifactName: "ollama-daemon-restart-unloaded", env: env(), timeoutMs: 90_000 }, + ); + expect(restart.exitCode, resultText(restart)).toBe(0); + const restartLines = restart.stdout.trim().split("\n"); + expect(restartLines[0]).toMatch(/^restart_mode=(system|user|manual)$/u); + expect(loadedOllamaModels(restartLines.slice(1).join("\n"))).toEqual([]); + + const recovered = await host.nemoclaw( + [ + SANDBOX_NAME, + "agent", + "--agent", + "main", + "--json", + "--session-id", + `e2e-gpu-ollama-restart-${Date.now()}-${process.pid}`, + "-m", + "Reply with exactly one word: PONG", + ], + { + artifactName: "agent-after-ollama-daemon-restart", + env: env(), + timeoutMs: 12 * 60_000, + }, + ); + expect(recovered.exitCode, resultText(recovered)).toBe(0); + expect(resultText(recovered)).toContain("Checking Ollama model readiness after daemon restart"); + expect(resultText(recovered)).toContain(`Ollama model '${model}' is loaded and ready.`); + expect(resultText(recovered)).toMatch(/pong/i); + + const loaded = await host.command("curl", ["-fsS", "http://127.0.0.1:11434/api/ps"], { + artifactName: "ollama-model-loaded-after-recovery", + env: env(), + timeoutMs: 30_000, + }); + expect(loaded.exitCode, resultText(loaded)).toBe(0); + expect(loadedOllamaModels(loaded.stdout)).toContain(model); }); From c3e13cbf2150b7d8092e761a7352d957f779ee67 Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 20:38:23 +0000 Subject: [PATCH 07/12] test(e2e): tolerate Ollama restart startup race Signed-off-by: cjagwani --- test/e2e/live/gpu-e2e.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index 85351d602f..ae700248ad 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -246,10 +246,13 @@ else restart_mode=manual fi for attempt in $(seq 1 60); do - if curl -fsS --connect-timeout 2 http://127.0.0.1:11434/api/tags >/dev/null; then - echo "restart_mode=$restart_mode" - curl -fsS http://127.0.0.1:11434/api/ps - exit 0 + tags_json="$(curl -fsS --connect-timeout 2 http://127.0.0.1:11434/api/tags 2>/dev/null || true)" + if [ -n "$tags_json" ]; then + ps_json="$(curl -fsS --connect-timeout 2 http://127.0.0.1:11434/api/ps 2>/dev/null || true)" + if [ -n "$ps_json" ]; then + printf 'restart_mode=%s\n%s\n' "$restart_mode" "$ps_json" + exit 0 + fi fi sleep 1 done From 444848f2a91c5884d05bdaf59daccd972c3a47ee Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 20:52:23 +0000 Subject: [PATCH 08/12] test(e2e): avoid GPU restart login shell Signed-off-by: cjagwani --- test/e2e/live/gpu-e2e.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index ae700248ad..f8801da49d 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -234,7 +234,7 @@ test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { const restart = await host.command( "bash", [ - "-lc", + "-c", `set -euo pipefail if sudo -n systemctl restart ollama 2>/dev/null; then restart_mode=system From 474321bff6b1962ab746e104075d7dcf811b4e71 Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 21:05:28 +0000 Subject: [PATCH 09/12] test(ollama): cover all recovery skip reasons Signed-off-by: cjagwani --- .../agent/passthrough-ollama-recovery.test.ts | 13 +++++--- .../agent/passthrough-ollama-recovery.ts | 31 +++++++++++++------ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts index 2c3415f6ed..c4f65b9756 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -81,17 +81,20 @@ describe("runOllamaRestartRecovery", () => { expect(writes.join("")).toContain(message); }); - it("reports an unreachable daemon without blocking dispatch", () => { + it.each([ + ["already-loaded", "Ollama model 'qwen3.6:35b' is already loaded"], + ["unreachable", "Ollama was unreachable during the restart check"], + ["missing-model", "No Ollama model is recorded for this sandbox"], + ["not-ollama", "Checking Ollama model readiness after daemon restart"], + ] as const)("handles the %s skip reason", (reason, message) => { const { writes, proc } = makeProcMock(); runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ kind: "skipped", - reason: "unreachable", + reason, })); - expect(writes.join("")).toContain( - "Ollama was unreachable during the restart check; continuing to OpenClaw dispatch", - ); + expect(writes.join("")).toContain(message); }); it("contains an unexpected recovery exception", () => { diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts index 41044fdab9..a4be760b23 100644 --- a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -51,16 +51,27 @@ function reportRecovery( return; } - if (result.reason === "already-loaded") { - proc.stderr.write(` Ollama model '${model}' is already loaded.\n`); - } else if (result.reason === "unreachable") { - proc.stderr.write( - " Ollama was unreachable during the restart check; continuing to OpenClaw dispatch.\n", - ); - } else if (result.reason === "missing-model") { - proc.stderr.write( - " No Ollama model is recorded for this sandbox; continuing to OpenClaw dispatch.\n", - ); + const reason = result.reason; + switch (reason) { + case "already-loaded": + proc.stderr.write(` Ollama model '${model}' is already loaded.\n`); + break; + case "unreachable": + proc.stderr.write( + " Ollama was unreachable during the restart check; continuing to OpenClaw dispatch.\n", + ); + break; + case "missing-model": + proc.stderr.write( + " No Ollama model is recorded for this sandbox; continuing to OpenClaw dispatch.\n", + ); + break; + case "not-ollama": + break; + default: { + const _exhaustive: never = reason; + return _exhaustive; + } } } From eba6b8e92bf2cd26a3dac3770c182720051324df Mon Sep 17 00:00:00 2001 From: cjagwani Date: Wed, 8 Jul 2026 21:07:51 +0000 Subject: [PATCH 10/12] fix(ollama): accept thinking-only warm responses Signed-off-by: cjagwani --- .../agent/ollama-restart-recovery.test.ts | 26 +++++++++++++++++++ .../sandbox/agent/ollama-restart-recovery.ts | 10 +++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts index ff42c2f16e..55712f5f5e 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -26,6 +26,11 @@ function getCommandUrl(command: readonly string[]): string { return command.find((arg) => arg.startsWith("http://")) ?? ""; } +function getCommandBody(command: readonly string[]): Record { + const dataIndex = command.indexOf("-d"); + return JSON.parse(command[dataIndex + 1] ?? "null") as Record; +} + describe("maybeWarmOllamaAfterDaemonRestart", () => { it("skips routes that are not local Ollama", () => { expect( @@ -61,6 +66,11 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, ); + expect(getCommandBody(runCaptureExImpl.mock.calls[0][0])).toMatchObject({ + model: "qwen3.6:35b", + stream: false, + think: false, + }); }); it("maps an auth-proxy route back to host loopback", () => { @@ -191,6 +201,22 @@ describe("maybeWarmOllamaAfterDaemonRestart", () => { ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "ollama-error" }); }); + it("accepts a completed thinking-only response from a thinking model", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ + stdout: JSON.stringify({ response: "", thinking: "The model is ready.", done: true }), + exitCode: 0, + timedOut: false, + }), + }, + ), + ).toEqual({ kind: "warmed", ok: true, timedOut: false }); + }); + it.each([ ["empty body", ""], ["malformed JSON", "not-json"], diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts index b6017f4c3d..e4cd06927b 100644 --- a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -138,6 +138,7 @@ function buildWarmCommand(model: string, hostname: string): string[] { model, prompt: "Hello, reply in less than 5 words", stream: false, + think: false, keep_alive: "15m", options: { num_predict: 16 }, }); @@ -164,15 +165,14 @@ function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid- done?: unknown; error?: unknown; response?: unknown; + thinking?: unknown; }; if (typeof parsed.error === "string" && parsed.error.trim() !== "") { return "ollama-error"; } - if ( - parsed.done !== true || - typeof parsed.response !== "string" || - parsed.response.trim() === "" - ) { + const response = typeof parsed.response === "string" ? parsed.response.trim() : ""; + const thinking = typeof parsed.thinking === "string" ? parsed.thinking.trim() : ""; + if (parsed.done !== true || (!response && !thinking)) { return "invalid-response"; } return "ok"; From 4bf4193bf5bfd67c21fbce905a28de6e3e278cf3 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 8 Jul 2026 14:31:34 -0700 Subject: [PATCH 11/12] test(e2e): verify Ollama recovery reply Parse assistant JSON so an echoed prompt cannot satisfy the recovery assertion. Align the docs with the per-dispatch readiness probe. Signed-off-by: Apurv Kumaria Co-authored-by: cjagwani Co-authored-by: Ho Lim --- docs/inference/use-local-inference.mdx | 2 +- test/e2e/live/gpu-e2e.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/inference/use-local-inference.mdx b/docs/inference/use-local-inference.mdx index abb73080bd..bb40d10208 100644 --- a/docs/inference/use-local-inference.mdx +++ b/docs/inference/use-local-inference.mdx @@ -92,7 +92,7 @@ If the selected model declares that it does not support tool calling, onboarding The validation also requires structured chat-completions tool calls. If the model leaks tool-call JSON as plain message text, onboarding stops so you can choose a model that returns tool calls in the expected response field. If a host-side validation probe times out, NemoClaw retries the Ollama tool-call validation with a larger timeout before failing the setup. -After an Ollama daemon restart, the first agent passthrough checks the registered route to see whether the selected model is still loaded. +Each Ollama-backed OpenClaw agent passthrough checks the registered route to see whether the selected model is still loaded. If the daemon is reachable but the model is unloaded, NemoClaw sends a bounded warm-up request before dispatching to OpenClaw. On WSL, if you choose the Windows-host Ollama path, NemoClaw uses `host.docker.internal:11434` and pulls missing models through the Ollama HTTP API instead of requiring the `ollama` CLI inside WSL. diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index f8801da49d..f08dffdf30 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -287,7 +287,7 @@ exit 1`, expect(recovered.exitCode, resultText(recovered)).toBe(0); expect(resultText(recovered)).toContain("Checking Ollama model readiness after daemon restart"); expect(resultText(recovered)).toContain(`Ollama model '${model}' is loaded and ready.`); - expect(resultText(recovered)).toMatch(/pong/i); + expect(chatContent(recovered.stdout)).toMatch(/pong/i); const loaded = await host.command("curl", ["-fsS", "http://127.0.0.1:11434/api/ps"], { artifactName: "ollama-model-loaded-after-recovery", From 4df81458b4dc9c6b1a38f8b3600124686258368a Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 8 Jul 2026 14:45:46 -0700 Subject: [PATCH 12/12] test(e2e): parse Ollama recovery payload Use the shared OpenClaw envelope parser proven against the failed-run artifact. Signed-off-by: Apurv Kumaria Co-authored-by: cjagwani Co-authored-by: Ho Lim --- test/e2e/live/gpu-e2e.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index f08dffdf30..e385691980 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -5,6 +5,7 @@ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { parseOpenClawAgentText } from "./common-egress-agent-helpers.ts"; import { assertGpuInstallProofs, assertNvidiaAvailable, @@ -287,7 +288,7 @@ exit 1`, expect(recovered.exitCode, resultText(recovered)).toBe(0); expect(resultText(recovered)).toContain("Checking Ollama model readiness after daemon restart"); expect(resultText(recovered)).toContain(`Ollama model '${model}' is loaded and ready.`); - expect(chatContent(recovered.stdout)).toMatch(/pong/i); + expect(parseOpenClawAgentText(recovered.stdout)).toMatch(/pong/i); const loaded = await host.command("curl", ["-fsS", "http://127.0.0.1:11434/api/ps"], { artifactName: "ollama-model-loaded-after-recovery",