From 659fffe4fa3e48ec31efadbe98441c68e6f2b3c9 Mon Sep 17 00:00:00 2001 From: Souvik Ghosh <138186578+souvikDevloper@users.noreply.github.com> Date: Sat, 4 Jul 2026 09:23:12 +0530 Subject: [PATCH 1/5] fix(status): probe inference.local route for cloud providers status and doctor reported inference healthy by probing the upstream provider endpoint directly, but the agent actually calls the in-sandbox inference.local route. When that route was broken while the upstream was reachable, both commands still reported healthy, contradicting connect. The inference.local gateway-chain subprobe added in #3265 was gated to ollama/vllm-local providers only. Extend it to every known provider in both the status snapshot and doctor so a broken inference.local route surfaces as a failing gateway check instead of a false all-clear. Fixes #6192 Signed-off-by: souvikDevloper --- src/lib/actions/sandbox/doctor-flow.test.ts | 57 ++++++++ src/lib/actions/sandbox/doctor.ts | 14 +- .../actions/sandbox/status-snapshot.test.ts | 122 ++++++++++++++++++ src/lib/actions/sandbox/status-snapshot.ts | 10 +- 4 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 src/lib/actions/sandbox/status-snapshot.test.ts diff --git a/src/lib/actions/sandbox/doctor-flow.test.ts b/src/lib/actions/sandbox/doctor-flow.test.ts index 86f20420a2..5ffd56638a 100644 --- a/src/lib/actions/sandbox/doctor-flow.test.ts +++ b/src/lib/actions/sandbox/doctor-flow.test.ts @@ -433,6 +433,63 @@ describe("runSandboxDoctor flow", () => { ); }); + it("probes the inference.local route for cloud providers, not just local ones (#6192)", async () => { + const harness = createDoctorHarness(); + // Cloud provider selected: registry + live `inference get` both report it. + harness.getSandboxSpy.mockReturnValue({ + name: "alpha", + agent: "openclaw", + model: "registry-model", + provider: "nvidia-prod", + openshellDriver: "docker", + gatewayName: "nemoclaw-19080", + gatewayPort: 19080, + }); + harness.captureOpenShellSpy.mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args : []; + if (argv[0] === "sandbox" && argv[1] === "list") { + return { status: 0, output: "alpha Ready" }; + } + if (argv[0] === "inference" && argv[1] === "get") { + return { status: 0, output: "Provider: nvidia-prod\nModel: nemotron\n" }; + } + return { status: 0, output: "" }; + }); + // The upstream provider endpoint is reachable... + harness.healthProbeSpy.mockReturnValue({ + ok: true, + probed: true, + providerLabel: "NVIDIA Cloud", + endpoint: "https://integrate.api.nvidia.com/v1/models", + detail: "reachable", + }); + // ...but the in-sandbox inference.local route the agent actually calls is + // broken (harness default: gateway probe resolves ok: false). + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + // Before #6192 this probe was gated to ollama/vllm-local; a cloud route + // never had inference.local checked, so a broken route read as healthy. + expect(harness.probeSandboxInferenceGatewayHealthSpy).toHaveBeenCalledWith("alpha"); + expect(report?.checks).toEqual( + expect.arrayContaining([ + // Upstream reachability stays green on its own... + expect.objectContaining({ + group: "Inference", + label: "Provider health", + status: "ok", + }), + // ...but the broken inference.local route now surfaces as a failure. + expect.objectContaining({ + group: "Inference", + label: "Provider health (gateway)", + status: "fail", + }), + ]), + ); + expect(report?.status).toBe("fail"); + }); + it("reports agent definition failures instead of hiding the runtime channel check", async () => { const harness = createDoctorHarness(); harness.configuredMessagingChannelsSpy.mockReturnValue(["telegram"]); diff --git a/src/lib/actions/sandbox/doctor.ts b/src/lib/actions/sandbox/doctor.ts index 5e8b244f05..6fb7cdafe2 100644 --- a/src/lib/actions/sandbox/doctor.ts +++ b/src/lib/actions/sandbox/doctor.ts @@ -326,10 +326,6 @@ function inferenceRouteCheck(sandboxName: string, route: InferenceRoute): Doctor }; } -function isLocalInferenceProvider(provider: string): boolean { - return provider === "ollama-local" || provider === "vllm-local"; -} - function skippedInferenceGatewayProbe(): ProviderHealthStatus { return { ok: false, @@ -341,13 +337,18 @@ function skippedInferenceGatewayProbe(): ProviderHealthStatus { }; } +// Probe the `inference.local` route the agent actually calls, regardless of +// provider. Cloud providers (nvidia-prod, etc.) route through the same +// in-sandbox openclaw gateway / auth proxy as local ones, so a reachable +// upstream endpoint does not prove the agent's route works. #3265 added this +// hop for ollama/vllm-local only; #6192 extends it to every provider so +// status/doctor stop reporting "healthy" when `inference.local` is broken. +// Unknown providers never reach here (collectInferenceChecks returns early). async function collectInferenceSubprobes( sandboxName: string, - provider: string, sandboxReachable: boolean, existing: ProviderHealthStatus[], ): Promise { - if (!isLocalInferenceProvider(provider)) return existing; if (!sandboxReachable) return [...existing, skippedInferenceGatewayProbe()]; const gateway = await probeSandboxInferenceGatewayHealth(sandboxName); if (!gateway) return existing; @@ -385,7 +386,6 @@ async function collectInferenceChecks( const subprobes = await collectInferenceSubprobes( sandboxName, - route.provider, sandboxReachable, health.subprobes ?? [], ); diff --git a/src/lib/actions/sandbox/status-snapshot.test.ts b/src/lib/actions/sandbox/status-snapshot.test.ts new file mode 100644 index 0000000000..be33718a74 --- /dev/null +++ b/src/lib/actions/sandbox/status-snapshot.test.ts @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { SandboxGatewayState } from "./gateway-state"; +import type { ProviderHealthStatus } from "../../inference/health"; + +type CollectSandboxStatusSnapshot = + typeof import("./status-snapshot")["collectSandboxStatusSnapshot"]; + +const requireDist = createRequire(import.meta.url); +const snapshotModulePath = "./status-snapshot.js"; + +// Warm the CommonJS source graph outside the first test's timeout. +requireDist(snapshotModulePath); +delete require.cache[requireDist.resolve(snapshotModulePath)]; + +const presentLookup: SandboxGatewayState = { + state: "present", + output: "Name: alpha\nPhase: Ready\n", +}; + +const reachableCloudHealth: ProviderHealthStatus = { + ok: true, + probed: true, + providerLabel: "NVIDIA Cloud", + endpoint: "https://integrate.api.nvidia.com/v1/models", + detail: "reachable", +}; + +function createSnapshotHarness(options: { + provider: string | null; + inferenceOutput: string; +}) { + delete require.cache[requireDist.resolve(snapshotModulePath)]; + + const runtime = requireDist("../../adapters/openshell/runtime.js"); + const processRecovery = requireDist("./process-recovery.js"); + + vi.spyOn(runtime, "captureOpenshellForStatus").mockResolvedValue({ + status: 0, + output: options.inferenceOutput, + }); + const probeSandboxInferenceGatewayHealthSpy = vi + .spyOn(processRecovery, "probeSandboxInferenceGatewayHealth") + .mockResolvedValue({ + ok: false, + endpoint: "https://inference.local/v1/models", + httpStatus: 0, + detail: "Inference gateway unreachable on https://inference.local/v1/models", + }); + + const sandboxEntry = { + name: "alpha", + agent: "openclaw", + model: "registry-model", + ...(options.provider ? { provider: options.provider } : {}), + openshellDriver: "docker", + }; + + const collectSandboxStatusSnapshot: CollectSandboxStatusSnapshot = + requireDist(snapshotModulePath).collectSandboxStatusSnapshot; + + return { + probeSandboxInferenceGatewayHealthSpy, + run: () => + collectSandboxStatusSnapshot("alpha", { + deps: { + getSandbox: (() => sandboxEntry) as never, + reconcile: async () => presentLookup, + // Upstream provider reachability is healthy on its own; the broken + // in-sandbox route must still be surfaced. + probeProviderHealthImpl: (provider: string) => + provider === "unknown" ? null : reachableCloudHealth, + }, + }), + }; +} + +describe("collectSandboxStatusSnapshot inference.local subprobe (#6192)", () => { + afterEach(() => { + vi.restoreAllMocks(); + delete require.cache[requireDist.resolve(snapshotModulePath)]; + }); + + it("probes the inference.local route for a cloud provider and reports it broken", async () => { + const harness = createSnapshotHarness({ + provider: "nvidia-prod", + inferenceOutput: "Provider: nvidia-prod\nModel: nemotron\n", + }); + + const snapshot = await harness.run(); + + // Before #6192 the gateway chain was probed only for ollama/vllm-local, so + // a cloud route never had inference.local checked. + expect(harness.probeSandboxInferenceGatewayHealthSpy).toHaveBeenCalledWith("alpha"); + expect(snapshot.currentProvider).toBe("nvidia-prod"); + expect(snapshot.inferenceHealth?.ok).toBe(true); + expect(snapshot.inferenceHealth?.subprobes).toEqual([ + expect.objectContaining({ + providerLabel: "Inference gateway chain", + probeLabel: "gateway", + ok: false, + }), + ]); + }); + + it("skips the inference.local subprobe when the provider is unknown", async () => { + const harness = createSnapshotHarness({ + provider: null, + inferenceOutput: "", + }); + + const snapshot = await harness.run(); + + expect(snapshot.currentProvider).toBe("unknown"); + expect(harness.probeSandboxInferenceGatewayHealthSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index dbef4db2fb..de99c5d22f 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -220,11 +220,11 @@ export async function collectSandboxStatusSnapshot( currentModel, opts.deps?.probeProviderHealthImpl, ); - if ( - inferenceHealth && - lookup.state === "present" && - (currentProvider === "ollama-local" || currentProvider === "vllm-local") - ) { + // Probe the `inference.local` route the agent actually calls for every known + // provider, not just local ones. Cloud providers route through the same + // in-sandbox gateway / auth proxy, so upstream reachability alone must not + // report the route as healthy when `inference.local` is broken (#6192). + if (inferenceHealth && lookup.state === "present" && currentProvider !== "unknown") { const gatewayChain = await probeSandboxInferenceGatewayHealth(sandboxName); if (gatewayChain) { const gatewaySubprobe: ProviderHealthStatus = { From d5bcbc7da4831c09ea6e818f19f0649b6ab631fd Mon Sep 17 00:00:00 2001 From: Souvik Ghosh <138186578+souvikDevloper@users.noreply.github.com> Date: Sat, 4 Jul 2026 09:23:13 +0530 Subject: [PATCH 2/5] test(doctor): keep inference.local cloud test linear The codebase-growth-guardrails check requires changed test files not to add if statements so test bodies stay linear. Replace the two-branch captureOpenshell mock in the new cloud-provider test with a keyed response lookup, dropping the added if statements without changing the assertions. Signed-off-by: souvikDevloper --- src/lib/actions/sandbox/doctor-flow.test.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/doctor-flow.test.ts b/src/lib/actions/sandbox/doctor-flow.test.ts index 5ffd56638a..e15703cb93 100644 --- a/src/lib/actions/sandbox/doctor-flow.test.ts +++ b/src/lib/actions/sandbox/doctor-flow.test.ts @@ -445,15 +445,13 @@ describe("runSandboxDoctor flow", () => { gatewayName: "nemoclaw-19080", gatewayPort: 19080, }); + const openShellResponses: Record = { + "sandbox list": { status: 0, output: "alpha Ready" }, + "inference get": { status: 0, output: "Provider: nvidia-prod\nModel: nemotron\n" }, + }; harness.captureOpenShellSpy.mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args : []; - if (argv[0] === "sandbox" && argv[1] === "list") { - return { status: 0, output: "alpha Ready" }; - } - if (argv[0] === "inference" && argv[1] === "get") { - return { status: 0, output: "Provider: nvidia-prod\nModel: nemotron\n" }; - } - return { status: 0, output: "" }; + return openShellResponses[`${argv[0]} ${argv[1]}`] ?? { status: 0, output: "" }; }); // The upstream provider endpoint is reachable... harness.healthProbeSpy.mockReturnValue({ From 9a3f8bda50b688a9908ae07d679b6264e086d5ef Mon Sep 17 00:00:00 2001 From: Souvik Ghosh <138186578+souvikDevloper@users.noreply.github.com> Date: Sat, 4 Jul 2026 09:23:13 +0530 Subject: [PATCH 3/5] refactor(inference): dedupe inference.local gateway subprobe Extract the duplicated inference.local gateway-chain subprobe object, built identically in doctor.ts and status-snapshot.ts, into a shared buildInferenceGatewaySubprobe helper in inference/health.ts so the subprobe shape stays in sync across both call sites. Also clone the shared provider-health fixture per invocation in the status-snapshot test: collectSandboxStatusSnapshot appends subprobes in place, so returning the module-level object would leak state across runs. Both changes address CodeRabbit review feedback on #6264; behavior and assertions are unchanged. Signed-off-by: souvikDevloper --- src/lib/actions/sandbox/doctor.ts | 19 ++++++----------- .../actions/sandbox/status-snapshot.test.ts | 6 ++++-- src/lib/actions/sandbox/status-snapshot.ts | 15 +++++-------- src/lib/inference/health.ts | 21 +++++++++++++++++++ 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/lib/actions/sandbox/doctor.ts b/src/lib/actions/sandbox/doctor.ts index 6fb7cdafe2..0e17ea304b 100644 --- a/src/lib/actions/sandbox/doctor.ts +++ b/src/lib/actions/sandbox/doctor.ts @@ -15,7 +15,11 @@ import { recoverNamedGatewayRuntime, } from "../../gateway-runtime-action"; import { parseGatewayInference } from "../../inference/config"; -import { type ProviderHealthStatus, probeProviderHealth } from "../../inference/health"; +import { + buildInferenceGatewaySubprobe, + type ProviderHealthStatus, + probeProviderHealth, +} from "../../inference/health"; import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; import { executeSandboxCommandForVerification } from "../../onboard/sandbox-verification-exec"; import { ROOT } from "../../runner"; @@ -352,18 +356,7 @@ async function collectInferenceSubprobes( if (!sandboxReachable) return [...existing, skippedInferenceGatewayProbe()]; const gateway = await probeSandboxInferenceGatewayHealth(sandboxName); if (!gateway) return existing; - return [ - ...existing, - { - ok: gateway.ok, - probed: true, - providerLabel: "Inference gateway chain", - endpoint: gateway.endpoint, - detail: gateway.detail, - probeLabel: "gateway", - ...(gateway.ok ? {} : { failureLabel: "unreachable" as const }), - }, - ]; + return [...existing, buildInferenceGatewaySubprobe(gateway)]; } async function collectInferenceChecks( diff --git a/src/lib/actions/sandbox/status-snapshot.test.ts b/src/lib/actions/sandbox/status-snapshot.test.ts index be33718a74..faa7d03e3c 100644 --- a/src/lib/actions/sandbox/status-snapshot.test.ts +++ b/src/lib/actions/sandbox/status-snapshot.test.ts @@ -72,9 +72,11 @@ function createSnapshotHarness(options: { getSandbox: (() => sandboxEntry) as never, reconcile: async () => presentLookup, // Upstream provider reachability is healthy on its own; the broken - // in-sandbox route must still be surfaced. + // in-sandbox route must still be surfaced. Return a fresh clone each + // call: collectSandboxStatusSnapshot appends `subprobes` in place, so + // sharing the module-level fixture would leak state across runs. probeProviderHealthImpl: (provider: string) => - provider === "unknown" ? null : reachableCloudHealth, + provider === "unknown" ? null : { ...reachableCloudHealth }, }, }), }; diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index de99c5d22f..c825409520 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -10,6 +10,7 @@ import { type AgentDefinition, getAgentRuntimeKind, loadAgent } from "../../agen import { withStdoutRedirectedToStderr } from "../../cli/stdout-guard"; import { parseGatewayInference } from "../../inference/config"; import { + buildInferenceGatewaySubprobe, type ProviderHealthProbeOptions, type ProviderHealthStatus, probeProviderHealth, @@ -227,16 +228,10 @@ export async function collectSandboxStatusSnapshot( if (inferenceHealth && lookup.state === "present" && currentProvider !== "unknown") { const gatewayChain = await probeSandboxInferenceGatewayHealth(sandboxName); if (gatewayChain) { - const gatewaySubprobe: ProviderHealthStatus = { - ok: gatewayChain.ok, - probed: true, - providerLabel: "Inference gateway chain", - endpoint: gatewayChain.endpoint, - detail: gatewayChain.detail, - probeLabel: "gateway", - ...(gatewayChain.ok ? {} : { failureLabel: "unreachable" as const }), - }; - inferenceHealth.subprobes = [...(inferenceHealth.subprobes ?? []), gatewaySubprobe]; + inferenceHealth.subprobes = [ + ...(inferenceHealth.subprobes ?? []), + buildInferenceGatewaySubprobe(gatewayChain), + ]; } } const statusAgent = resolveSandboxStatusAgent(sb?.agent || "openclaw"); diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index 4d05cc3f92..dce5a4a9b0 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -38,6 +38,27 @@ export interface ProviderHealthStatus { subprobes?: ProviderHealthStatus[]; } +/** + * Build the `inference.local` gateway-chain subprobe rendered alongside the + * provider health probe. Shared by `status` and `doctor` so the subprobe shape + * stays in sync across both call sites (#3265, #6192). + */ +export function buildInferenceGatewaySubprobe(gateway: { + ok: boolean; + endpoint: string; + detail: string; +}): ProviderHealthStatus { + return { + ok: gateway.ok, + probed: true, + providerLabel: "Inference gateway chain", + endpoint: gateway.endpoint, + detail: gateway.detail, + probeLabel: "gateway", + ...(gateway.ok ? {} : { failureLabel: "unreachable" as const }), + }; +} + export interface ProviderHealthProbeOptions { runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; model?: string | null; From 1b01a58c0d4324f3a71058d130a33b7171b910f9 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 7 Jul 2026 01:02:57 -0700 Subject: [PATCH 4/5] chore(sandbox): format status snapshot test --- src/lib/actions/sandbox/status-snapshot.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lib/actions/sandbox/status-snapshot.test.ts b/src/lib/actions/sandbox/status-snapshot.test.ts index faa7d03e3c..c4ed739bce 100644 --- a/src/lib/actions/sandbox/status-snapshot.test.ts +++ b/src/lib/actions/sandbox/status-snapshot.test.ts @@ -31,10 +31,7 @@ const reachableCloudHealth: ProviderHealthStatus = { detail: "reachable", }; -function createSnapshotHarness(options: { - provider: string | null; - inferenceOutput: string; -}) { +function createSnapshotHarness(options: { provider: string | null; inferenceOutput: string }) { delete require.cache[requireDist.resolve(snapshotModulePath)]; const runtime = requireDist("../../adapters/openshell/runtime.js"); From ead75a2391edf00a95c3576968f5ab74cacb8428 Mon Sep 17 00:00:00 2001 From: Souvik Ghosh <138186578+souvikDevloper@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:56:16 +0530 Subject: [PATCH 5/5] fix(status): degrade inference aggregate when inference.local route is broken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the inference.local gateway subprobe failed, status left the aggregate inferenceHealth.ok true, so `status` (text and --json) could still report inference healthy while `connect` reports the route BROKEN — the exact contradiction #6192 set out to fix. Propagate the subprobe failure into the aggregate (ok=false + failureLabel + explanatory detail) so the top-level status reflects the route the agent actually uses. Rework the status-snapshot test onto typed dependency injection (injectable captureLiveInference + probeInferenceGateway) instead of the createRequire seam, keeping it within the CLI createRequire budget, and add regression coverage for the degraded aggregate (surfaced verbatim in --json) and the degraded text output. Addresses review feedback on #6264. Signed-off-by: Souvik Ghosh <138186578+souvikDevloper@users.noreply.github.com> --- src/lib/actions/sandbox/status-flow.test.ts | 40 +++++ .../actions/sandbox/status-snapshot.test.ts | 143 ++++++++---------- src/lib/actions/sandbox/status-snapshot.ts | 28 +++- 3 files changed, 127 insertions(+), 84 deletions(-) diff --git a/src/lib/actions/sandbox/status-flow.test.ts b/src/lib/actions/sandbox/status-flow.test.ts index 5d0b663d2d..ee4b3e3674 100644 --- a/src/lib/actions/sandbox/status-flow.test.ts +++ b/src/lib/actions/sandbox/status-flow.test.ts @@ -138,6 +138,46 @@ describe("showSandboxStatus flow", () => { expect(output).toContain("http://127.0.0.1:11434/api/tags"); }); + it("reports inference unhealthy in text when the inference.local route is broken (#6192)", async () => { + // Mirrors what collectSandboxStatusSnapshot now produces for a cloud + // provider whose upstream endpoint is reachable but whose inference.local + // route is broken: the aggregate is degraded so the top-level line does not + // render a bare "healthy" that contradicts `connect`. + const harness = createStatusFlowHarness({ + currentProvider: "nvidia-prod", + inferenceHealth: { + ok: false, + probed: true, + providerLabel: "NVIDIA Cloud", + endpoint: "https://integrate.api.nvidia.com/v1/models", + detail: + "Provider endpoint is reachable, but the inference.local route the agent uses is broken. " + + "Inference gateway unreachable on https://inference.local/v1/models", + failureLabel: "unreachable", + subprobes: [ + { + ok: false, + probed: true, + providerLabel: "Inference gateway chain", + endpoint: "https://inference.local/v1/models", + detail: "Inference gateway unreachable on https://inference.local/v1/models", + probeLabel: "gateway", + failureLabel: "unreachable", + }, + ], + }, + sandboxEntry: { provider: "nvidia-prod" }, + }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).not.toContain("Inference: healthy"); + expect(output).toContain("Inference: unreachable"); + expect(output).toContain("Inference (gateway):"); + expect(output).toContain("the inference.local route the agent uses is broken"); + }); + it("renders fresh shields posture as not configured rather than down", async () => { const harness = createStatusFlowHarness({ shieldsPosture: { diff --git a/src/lib/actions/sandbox/status-snapshot.test.ts b/src/lib/actions/sandbox/status-snapshot.test.ts index c4ed739bce..f797494647 100644 --- a/src/lib/actions/sandbox/status-snapshot.test.ts +++ b/src/lib/actions/sandbox/status-snapshot.test.ts @@ -1,29 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createRequire } from "node:module"; +import { describe, expect, it, vi } from "vitest"; -import { afterEach, describe, expect, it, vi } from "vitest"; - -import type { SandboxGatewayState } from "./gateway-state"; import type { ProviderHealthStatus } from "../../inference/health"; - -type CollectSandboxStatusSnapshot = - typeof import("./status-snapshot")["collectSandboxStatusSnapshot"]; - -const requireDist = createRequire(import.meta.url); -const snapshotModulePath = "./status-snapshot.js"; - -// Warm the CommonJS source graph outside the first test's timeout. -requireDist(snapshotModulePath); -delete require.cache[requireDist.resolve(snapshotModulePath)]; +import type { SandboxGatewayState } from "./gateway-state"; +import { collectSandboxStatusSnapshot } from "./status-snapshot"; const presentLookup: SandboxGatewayState = { state: "present", output: "Name: alpha\nPhase: Ready\n", }; -const reachableCloudHealth: ProviderHealthStatus = { +// Upstream provider endpoint is reachable on its own; a broken in-sandbox +// route must still be surfaced. +const reachableProviderHealth: ProviderHealthStatus = { ok: true, probed: true, providerLabel: "NVIDIA Cloud", @@ -31,25 +22,7 @@ const reachableCloudHealth: ProviderHealthStatus = { detail: "reachable", }; -function createSnapshotHarness(options: { provider: string | null; inferenceOutput: string }) { - delete require.cache[requireDist.resolve(snapshotModulePath)]; - - const runtime = requireDist("../../adapters/openshell/runtime.js"); - const processRecovery = requireDist("./process-recovery.js"); - - vi.spyOn(runtime, "captureOpenshellForStatus").mockResolvedValue({ - status: 0, - output: options.inferenceOutput, - }); - const probeSandboxInferenceGatewayHealthSpy = vi - .spyOn(processRecovery, "probeSandboxInferenceGatewayHealth") - .mockResolvedValue({ - ok: false, - endpoint: "https://inference.local/v1/models", - httpStatus: 0, - detail: "Inference gateway unreachable on https://inference.local/v1/models", - }); - +function runSnapshot(options: { provider: string | null; gatewayOk: boolean }) { const sandboxEntry = { name: "alpha", agent: "openclaw", @@ -57,48 +30,57 @@ function createSnapshotHarness(options: { provider: string | null; inferenceOutp ...(options.provider ? { provider: options.provider } : {}), openshellDriver: "docker", }; - - const collectSandboxStatusSnapshot: CollectSandboxStatusSnapshot = - requireDist(snapshotModulePath).collectSandboxStatusSnapshot; - - return { - probeSandboxInferenceGatewayHealthSpy, - run: () => - collectSandboxStatusSnapshot("alpha", { - deps: { - getSandbox: (() => sandboxEntry) as never, - reconcile: async () => presentLookup, - // Upstream provider reachability is healthy on its own; the broken - // in-sandbox route must still be surfaced. Return a fresh clone each - // call: collectSandboxStatusSnapshot appends `subprobes` in place, so - // sharing the module-level fixture would leak state across runs. - probeProviderHealthImpl: (provider: string) => - provider === "unknown" ? null : { ...reachableCloudHealth }, + const probeInferenceGateway = vi.fn(async () => + options.gatewayOk + ? { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "Inference gateway responded HTTP 200 (full chain reachable).", + } + : { + ok: false, + endpoint: "https://inference.local/v1/models", + httpStatus: 0, + detail: "Inference gateway unreachable on https://inference.local/v1/models", }, - }), - }; + ); + const snapshot = collectSandboxStatusSnapshot("alpha", { + deps: { + getSandbox: (() => sandboxEntry) as never, + reconcile: async () => presentLookup, + // Live `inference get` reports the active provider so the snapshot picks + // the cloud route without spawning openshell. + captureLiveInference: (async () => ({ + status: 0, + output: options.provider ? `Provider: ${options.provider}\nModel: nemotron\n` : "", + })) as never, + probeInferenceGateway: probeInferenceGateway as never, + // A fresh clone per call: the snapshot mutates `subprobes`/`ok` in place. + probeProviderHealthImpl: (provider: string) => + provider === "unknown" ? null : { ...reachableProviderHealth }, + }, + }); + return { probeInferenceGateway, snapshot }; } describe("collectSandboxStatusSnapshot inference.local subprobe (#6192)", () => { - afterEach(() => { - vi.restoreAllMocks(); - delete require.cache[requireDist.resolve(snapshotModulePath)]; - }); - - it("probes the inference.local route for a cloud provider and reports it broken", async () => { - const harness = createSnapshotHarness({ + it("degrades the aggregate to unhealthy when the inference.local route is broken", async () => { + const { probeInferenceGateway, snapshot } = runSnapshot({ provider: "nvidia-prod", - inferenceOutput: "Provider: nvidia-prod\nModel: nemotron\n", + gatewayOk: false, }); - - const snapshot = await harness.run(); - - // Before #6192 the gateway chain was probed only for ollama/vllm-local, so - // a cloud route never had inference.local checked. - expect(harness.probeSandboxInferenceGatewayHealthSpy).toHaveBeenCalledWith("alpha"); - expect(snapshot.currentProvider).toBe("nvidia-prod"); - expect(snapshot.inferenceHealth?.ok).toBe(true); - expect(snapshot.inferenceHealth?.subprobes).toEqual([ + const result = await snapshot; + + // Cloud providers now have the route the agent actually uses probed. + expect(probeInferenceGateway).toHaveBeenCalledWith("alpha"); + expect(result.currentProvider).toBe("nvidia-prod"); + // The aggregate is surfaced verbatim as `report.inferenceHealth` in --json + // and drives the text summary. It must NOT stay healthy when the real route + // is broken, even though the upstream provider endpoint is reachable. + expect(result.inferenceHealth?.ok).toBe(false); + expect(result.inferenceHealth?.failureLabel).toBe("unreachable"); + expect(result.inferenceHealth?.subprobes).toEqual([ expect.objectContaining({ providerLabel: "Inference gateway chain", probeLabel: "gateway", @@ -107,15 +89,20 @@ describe("collectSandboxStatusSnapshot inference.local subprobe (#6192)", () => ]); }); - it("skips the inference.local subprobe when the provider is unknown", async () => { - const harness = createSnapshotHarness({ - provider: null, - inferenceOutput: "", - }); + it("keeps the aggregate healthy when the inference.local route is reachable", async () => { + const { snapshot } = runSnapshot({ provider: "nvidia-prod", gatewayOk: true }); + const result = await snapshot; - const snapshot = await harness.run(); + expect(result.inferenceHealth?.ok).toBe(true); + expect(result.inferenceHealth?.failureLabel).toBeUndefined(); + expect(result.inferenceHealth?.subprobes?.[0]?.ok).toBe(true); + }); + + it("skips the inference.local subprobe when the provider is unknown", async () => { + const { probeInferenceGateway, snapshot } = runSnapshot({ provider: null, gatewayOk: false }); + const result = await snapshot; - expect(snapshot.currentProvider).toBe("unknown"); - expect(harness.probeSandboxInferenceGatewayHealthSpy).not.toHaveBeenCalled(); + expect(result.currentProvider).toBe("unknown"); + expect(probeInferenceGateway).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index c825409520..3393dff5dd 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -157,6 +157,8 @@ interface CollectSandboxStatusSnapshotDeps { probeProviderHealthImpl?: ProbeProviderHealth; probeTerminalRuntimeHealth?: ProbeTerminalRuntimeHealth; reconcile?: ReconcileSandboxGatewayState; + captureLiveInference?: typeof captureOpenshellForStatus; + probeInferenceGateway?: typeof probeSandboxInferenceGatewayHealth; } export async function collectSandboxStatusSnapshot( @@ -173,6 +175,9 @@ export async function collectSandboxStatusSnapshot( getState: getSandboxGatewayStateForStatus, })); const getSandbox = opts.deps?.getSandbox ?? registry.getSandbox; + const captureLiveInference = opts.deps?.captureLiveInference ?? captureOpenshellForStatus; + const probeInferenceGateway = + opts.deps?.probeInferenceGateway ?? probeSandboxInferenceGatewayHealth; const sb = getSandbox(sandboxName); let lookup: SandboxGatewayState; try { @@ -187,7 +192,7 @@ export async function collectSandboxStatusSnapshot( let liveResult: Awaited> | null = null; if (lookup.state === "present") { try { - liveResult = await captureOpenshellForStatus(["inference", "get"]); + liveResult = await captureLiveInference(["inference", "get"]); } catch { liveResult = null; } @@ -226,12 +231,23 @@ export async function collectSandboxStatusSnapshot( // in-sandbox gateway / auth proxy, so upstream reachability alone must not // report the route as healthy when `inference.local` is broken (#6192). if (inferenceHealth && lookup.state === "present" && currentProvider !== "unknown") { - const gatewayChain = await probeSandboxInferenceGatewayHealth(sandboxName); + const gatewayChain = await probeInferenceGateway(sandboxName); if (gatewayChain) { - inferenceHealth.subprobes = [ - ...(inferenceHealth.subprobes ?? []), - buildInferenceGatewaySubprobe(gatewayChain), - ]; + const gatewaySubprobe = buildInferenceGatewaySubprobe(gatewayChain); + inferenceHealth.subprobes = [...(inferenceHealth.subprobes ?? []), gatewaySubprobe]; + if (!gatewaySubprobe.ok) { + // The agent's real route (`inference.local`) is broken even though the + // upstream provider endpoint is reachable. The aggregate must not stay + // `ok: true`, or `status` (text and --json) would report healthy while + // `connect` reports the route BROKEN — the exact contradiction #6192 + // set out to fix. Degrade the top-level result and explain the cause. + inferenceHealth.ok = false; + inferenceHealth.failureLabel = + inferenceHealth.failureLabel ?? gatewaySubprobe.failureLabel ?? "unreachable"; + inferenceHealth.detail = + "Provider endpoint is reachable, but the inference.local route the agent " + + `uses is broken. ${gatewaySubprobe.detail}`; + } } } const statusAgent = resolveSandboxStatusAgent(sb?.agent || "openclaw");