From a74eb6800cec7877b75256f1802b05198eecf0ef Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Mon, 6 Jul 2026 22:08:46 +0800 Subject: [PATCH 1/9] fix: route loopback compatible endpoints through gateway host --- docs/inference/use-local-inference.mdx | 2 + .../inference-providers/remote.test.ts | 92 +++++++++++++++++++ src/lib/onboard/inference-providers/remote.ts | 28 +++++- 3 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 src/lib/onboard/inference-providers/remote.test.ts diff --git a/docs/inference/use-local-inference.mdx b/docs/inference/use-local-inference.mdx index bd80cb4e84..3fd2600017 100644 --- a/docs/inference/use-local-inference.mdx +++ b/docs/inference/use-local-inference.mdx @@ -245,6 +245,8 @@ NEMOCLAW_PROVIDER=custom \ ``` Use the model id reported by the server's `/v1/models` response. +For loopback HTTP endpoints such as `http://localhost:8000/v1`, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` so sandbox traffic can leave the container namespace. +Make sure the local server listens on an address reachable from containers, such as `0.0.0.0`; a server bound only to `127.0.0.1` can still be unreachable from the sandbox route. If your setup uses the legacy containerized OpenShell gateway path instead of the default host-process gateway, the reachable host-service URL can differ; `http://host.openshell.internal:8000/v1` is the usual legacy host-gateway route and port `8000` is included in NemoClaw's `local-inference` policy preset. After onboarding, run `$$nemoclaw status` and check the `Inference` row before starting long agent work. diff --git a/src/lib/onboard/inference-providers/remote.test.ts b/src/lib/onboard/inference-providers/remote.test.ts new file mode 100644 index 0000000000..d1b44d9ec6 --- /dev/null +++ b/src/lib/onboard/inference-providers/remote.test.ts @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { setupRemoteProviderInference } from "./remote"; +import type { RemoteProviderDeps } from "./types"; + +class ExitError extends Error { + constructor(readonly code: number) { + super(`EXIT_CALLED:${code}`); + } +} + +function makeDeps(overrides: Partial = {}): RemoteProviderDeps { + return { + runOpenshell: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), + upsertProvider: vi.fn(() => ({ ok: true })), + verifyInferenceRoute: vi.fn(), + verifyOnboardInferenceSmoke: vi.fn(), + isNonInteractive: vi.fn(() => true), + registry: { updateSandbox: vi.fn() as never }, + exitProcess: vi.fn((code: number): never => { + throw new ExitError(code); + }), + error: vi.fn(), + log: vi.fn(), + REMOTE_PROVIDER_CONFIG: { + custom: { + label: "Other OpenAI-compatible endpoint", + providerName: "compatible-endpoint", + providerType: "openai", + credentialEnv: "COMPATIBLE_API_KEY", + endpointUrl: "https://example.invalid/v1", + helpUrl: null, + modelMode: "input", + defaultModel: "custom-model", + }, + }, + hydrateCredentialEnv: vi.fn(() => "dummy-key"), + promptValidationRecovery: vi.fn(async () => "selection" as const), + classifyApplyFailure: vi.fn(() => ({ kind: "unknown" }) as never), + LOCAL_INFERENCE_TIMEOUT_SECS: 180, + bedrockRuntimeOnboard: { + setupBedrockRuntimeInference: vi.fn(async () => ({ handled: false }) as const), + }, + redact: vi.fn((input: string) => input), + compactText: vi.fn((input: string) => input.trim()), + ...overrides, + }; +} + +describe("setupRemoteProviderInference", () => { + it("uses a sandbox-facing host alias for loopback compatible endpoints (#5744)", async () => { + const deps = makeDeps(); + + await expect( + setupRemoteProviderInference( + { + sandboxName: "dcode-vllm-local", + model: "Qwen/Qwen2.5-1.5B-Instruct", + provider: "compatible-endpoint", + endpointUrl: "http://localhost:8000/v1", + credentialEnv: "COMPATIBLE_API_KEY", + }, + deps, + ), + ).resolves.toEqual({ done: false }); + + expect(deps.upsertProvider).toHaveBeenCalledWith( + "compatible-endpoint", + "openai", + "COMPATIBLE_API_KEY", + "http://host.openshell.internal:8000/v1", + { COMPATIBLE_API_KEY: "dummy-key" }, + ); + expect(deps.runOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + "--no-verify", + "--provider", + "compatible-endpoint", + "--model", + "Qwen/Qwen2.5-1.5B-Instruct", + "--timeout", + "180", + ], + { ignoreError: true }, + ); + }); +}); diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index 1dd249be78..79179a96fb 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -8,6 +8,28 @@ import type { RemoteProviderDeps, SetupInferenceResult } from "./types"; +function gatewayReachableCompatibleEndpointUrl( + provider: string, + endpointUrl: string | null | undefined, +): string | null | undefined { + if (provider !== "compatible-endpoint" || !endpointUrl) return endpointUrl; + let parsed: URL; + try { + parsed = new URL(endpointUrl); + } catch { + return endpointUrl; + } + const hostname = parsed.hostname.toLowerCase(); + const isLoopback = + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]"; + if (parsed.protocol !== "http:" || !isLoopback || !parsed.port) return endpointUrl; + parsed.hostname = "host.openshell.internal"; + return parsed.toString().replace(/\/$/, parsed.pathname === "/" ? "/" : ""); +} + /** * Returns `{ done: true, result }` when the flow handled the request * (e.g. Bedrock short-circuit or a retry-to-selection); returns @@ -82,6 +104,7 @@ export async function setupRemoteProviderInference( const resolvedCredentialEnv = credentialEnv || (config && config.credentialEnv); const resolvedEndpointUrl = endpointUrl || (config && config.endpointUrl); let providerResult; + let gatewayEndpointUrl: string | null | undefined = resolvedEndpointUrl; if (reuseGatewayCredentialWithoutLocalKey) { // This is only a last-moment existence probe. The primary authorization // of the provider's non-secret credential/config binding identity is @@ -104,12 +127,13 @@ export async function setupRemoteProviderInference( resolvedCredentialEnv && credentialValue ? { [resolvedCredentialEnv]: credentialValue } : {}; + gatewayEndpointUrl = gatewayReachableCompatibleEndpointUrl(provider, resolvedEndpointUrl); providerResult = credentialValue ? upsertProvider( provider, config.providerType, resolvedCredentialEnv, - resolvedEndpointUrl, + gatewayEndpointUrl, env, ) : { @@ -138,7 +162,7 @@ export async function setupRemoteProviderInference( return exitProcess(providerResult.status || 1); } const argsv = ["inference", "set"]; - if (config.skipVerify) { + if (config.skipVerify || gatewayEndpointUrl !== resolvedEndpointUrl) { argsv.push("--no-verify"); } argsv.push("--provider", provider, "--model", model); From 030c98d6250e26954fbba11f0bc711b199305fcb Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Tue, 7 Jul 2026 00:42:50 +0800 Subject: [PATCH 2/9] fix(inference): tighten compatible endpoint gateway rewrite Signed-off-by: Chengjie Wang --- .../inference-providers/remote.test.ts | 106 +++++++++++++++++- src/lib/onboard/inference-providers/remote.ts | 30 +++-- 2 files changed, 124 insertions(+), 12 deletions(-) diff --git a/src/lib/onboard/inference-providers/remote.test.ts b/src/lib/onboard/inference-providers/remote.test.ts index d1b44d9ec6..1d6067ef64 100644 --- a/src/lib/onboard/inference-providers/remote.test.ts +++ b/src/lib/onboard/inference-providers/remote.test.ts @@ -36,6 +36,16 @@ function makeDeps(overrides: Partial = {}): RemoteProviderDe modelMode: "input", defaultModel: "custom-model", }, + openai: { + label: "OpenAI API", + providerName: "openai-api", + providerType: "openai", + credentialEnv: "OPENAI_API_KEY", + endpointUrl: "https://api.openai.com/v1", + helpUrl: null, + modelMode: "input", + defaultModel: "gpt-4.1", + }, }, hydrateCredentialEnv: vi.fn(() => "dummy-key"), promptValidationRecovery: vi.fn(async () => "selection" as const), @@ -51,6 +61,8 @@ function makeDeps(overrides: Partial = {}): RemoteProviderDe } describe("setupRemoteProviderInference", () => { + const model = "Qwen/Qwen2.5-1.5B-Instruct"; + it("uses a sandbox-facing host alias for loopback compatible endpoints (#5744)", async () => { const deps = makeDeps(); @@ -58,7 +70,7 @@ describe("setupRemoteProviderInference", () => { setupRemoteProviderInference( { sandboxName: "dcode-vllm-local", - model: "Qwen/Qwen2.5-1.5B-Instruct", + model, provider: "compatible-endpoint", endpointUrl: "http://localhost:8000/v1", credentialEnv: "COMPATIBLE_API_KEY", @@ -82,11 +94,101 @@ describe("setupRemoteProviderInference", () => { "--provider", "compatible-endpoint", "--model", - "Qwen/Qwen2.5-1.5B-Instruct", + model, "--timeout", "180", ], { ignoreError: true }, ); }); + + it("uses the gateway alias when reusing an existing loopback compatible provider (#5744)", async () => { + const deps = makeDeps(); + + await expect( + setupRemoteProviderInference( + { + sandboxName: "dcode-vllm-local", + model, + provider: "compatible-endpoint", + endpointUrl: "http://127.0.0.1:8000/v1", + credentialEnv: "COMPATIBLE_API_KEY", + reuseGatewayCredentialWithoutLocalKey: true, + }, + deps, + ), + ).resolves.toEqual({ done: false }); + + expect(deps.upsertProvider).not.toHaveBeenCalled(); + expect(deps.runOpenshell).toHaveBeenNthCalledWith( + 1, + ["provider", "get", "compatible-endpoint"], + { + ignoreError: true, + suppressOutput: true, + }, + ); + expect(deps.runOpenshell).toHaveBeenNthCalledWith( + 2, + [ + "inference", + "set", + "--no-verify", + "--provider", + "compatible-endpoint", + "--model", + model, + "--timeout", + "180", + ], + { ignoreError: true }, + ); + }); + + it.each([ + ["HTTPS loopback", "compatible-endpoint", "https://localhost:8000/v1"], + ["non-loopback host", "compatible-endpoint", "http://10.0.0.1:8000/v1"], + ["missing explicit port", "compatible-endpoint", "http://localhost/v1"], + ["privileged port", "compatible-endpoint", "http://localhost:1023/v1"], + ["embedded URL credentials", "compatible-endpoint", "http://user:pass@localhost:8000/v1"], + ["malformed URL", "compatible-endpoint", "not a url"], + ["non-compatible provider", "openai-api", "http://localhost:8000/v1"], + ])("does not rewrite %s endpoints", async (_name, provider, endpointUrl) => { + const deps = makeDeps(); + const credentialEnv = + provider === "compatible-endpoint" ? "COMPATIBLE_API_KEY" : "OPENAI_API_KEY"; + + await expect( + setupRemoteProviderInference( + { + sandboxName: "dcode-vllm-local", + model, + provider, + endpointUrl, + credentialEnv, + }, + deps, + ), + ).resolves.toEqual({ done: false }); + + expect(deps.upsertProvider).toHaveBeenCalledWith( + provider, + "openai", + credentialEnv, + endpointUrl, + { [credentialEnv]: "dummy-key" }, + ); + expect(deps.runOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + "--provider", + provider, + "--model", + model, + ...(provider === "compatible-endpoint" ? ["--timeout", "180"] : []), + ], + { ignoreError: true }, + ); + }); }); diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index 79179a96fb..9e7bc85f44 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -19,15 +19,23 @@ function gatewayReachableCompatibleEndpointUrl( } catch { return endpointUrl; } - const hostname = parsed.hostname.toLowerCase(); - const isLoopback = - hostname === "localhost" || - hostname === "127.0.0.1" || - hostname === "::1" || - hostname === "[::1]"; - if (parsed.protocol !== "http:" || !isLoopback || !parsed.port) return endpointUrl; + const hostname = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase(); + const port = Number(parsed.port); + const isLoopback = hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1"; + if ( + parsed.protocol !== "http:" || + parsed.username || + parsed.password || + !isLoopback || + !Number.isInteger(port) || + port < 1024 + ) { + return endpointUrl; + } parsed.hostname = "host.openshell.internal"; - return parsed.toString().replace(/\/$/, parsed.pathname === "/" ? "/" : ""); + const pathname = parsed.pathname.replace(/\/+$/, ""); + parsed.pathname = pathname || "/"; + return parsed.pathname === "/" ? parsed.origin : `${parsed.origin}${parsed.pathname}`; } /** @@ -103,8 +111,8 @@ export async function setupRemoteProviderInference( while (true) { const resolvedCredentialEnv = credentialEnv || (config && config.credentialEnv); const resolvedEndpointUrl = endpointUrl || (config && config.endpointUrl); + const gatewayEndpointUrl = gatewayReachableCompatibleEndpointUrl(provider, resolvedEndpointUrl); let providerResult; - let gatewayEndpointUrl: string | null | undefined = resolvedEndpointUrl; if (reuseGatewayCredentialWithoutLocalKey) { // This is only a last-moment existence probe. The primary authorization // of the provider's non-secret credential/config binding identity is @@ -127,7 +135,6 @@ export async function setupRemoteProviderInference( resolvedCredentialEnv && credentialValue ? { [resolvedCredentialEnv]: credentialValue } : {}; - gatewayEndpointUrl = gatewayReachableCompatibleEndpointUrl(provider, resolvedEndpointUrl); providerResult = credentialValue ? upsertProvider( provider, @@ -163,6 +170,9 @@ export async function setupRemoteProviderInference( } const argsv = ["inference", "set"]; if (config.skipVerify || gatewayEndpointUrl !== resolvedEndpointUrl) { + // The host-side probe validates the user-provided loopback URL. When the + // provider route is rewritten to host.openshell.internal, OpenShell's + // host-side verifier cannot resolve that sandbox-only bridge name. argsv.push("--no-verify"); } argsv.push("--provider", provider, "--model", model); From 14f49ff89c2aec4b8c8aaa9987dd7ee3eb835125 Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Tue, 7 Jul 2026 01:06:16 +0800 Subject: [PATCH 3/9] test(inference): cover compatible endpoint boundaries Signed-off-by: Chengjie Wang --- .../inference-providers/remote.test.ts | 94 +++++++++++++++++++ src/lib/onboard/inference-providers/remote.ts | 3 + 2 files changed, 97 insertions(+) diff --git a/src/lib/onboard/inference-providers/remote.test.ts b/src/lib/onboard/inference-providers/remote.test.ts index 1d6067ef64..ea9c912baa 100644 --- a/src/lib/onboard/inference-providers/remote.test.ts +++ b/src/lib/onboard/inference-providers/remote.test.ts @@ -35,6 +35,9 @@ function makeDeps(overrides: Partial = {}): RemoteProviderDe helpUrl: null, modelMode: "input", defaultModel: "custom-model", + // Mirrors production: compatible endpoints already skip OpenShell's + // verifier; these tests focus on which base URL gets registered. + skipVerify: true, }, openai: { label: "OpenAI API", @@ -102,6 +105,52 @@ describe("setupRemoteProviderInference", () => { ); }); + it.each([ + ["IPv6 loopback", "http://[::1]:8000/v1", "http://host.openshell.internal:8000/v1"], + [ + "first unprivileged port", + "http://localhost:1024/v1", + "http://host.openshell.internal:1024/v1", + ], + ])("rewrites %s compatible endpoint routes (#5744)", async (_name, endpointUrl, expectedUrl) => { + const deps = makeDeps(); + + await expect( + setupRemoteProviderInference( + { + sandboxName: "dcode-vllm-local", + model, + provider: "compatible-endpoint", + endpointUrl, + credentialEnv: "COMPATIBLE_API_KEY", + }, + deps, + ), + ).resolves.toEqual({ done: false }); + + expect(deps.upsertProvider).toHaveBeenCalledWith( + "compatible-endpoint", + "openai", + "COMPATIBLE_API_KEY", + expectedUrl, + { COMPATIBLE_API_KEY: "dummy-key" }, + ); + expect(deps.runOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + "--no-verify", + "--provider", + "compatible-endpoint", + "--model", + model, + "--timeout", + "180", + ], + { ignoreError: true }, + ); + }); + it("uses the gateway alias when reusing an existing loopback compatible provider (#5744)", async () => { const deps = makeDeps(); @@ -145,11 +194,55 @@ describe("setupRemoteProviderInference", () => { ); }); + it("keeps the original provider route when reusing an existing non-loopback compatible provider (#5744)", async () => { + const deps = makeDeps(); + + await expect( + setupRemoteProviderInference( + { + sandboxName: "dcode-vllm-local", + model, + provider: "compatible-endpoint", + endpointUrl: "http://10.0.0.1:8000/v1", + credentialEnv: "COMPATIBLE_API_KEY", + reuseGatewayCredentialWithoutLocalKey: true, + }, + deps, + ), + ).resolves.toEqual({ done: false }); + + expect(deps.upsertProvider).not.toHaveBeenCalled(); + expect(deps.runOpenshell).toHaveBeenNthCalledWith( + 1, + ["provider", "get", "compatible-endpoint"], + { + ignoreError: true, + suppressOutput: true, + }, + ); + expect(deps.runOpenshell).toHaveBeenNthCalledWith( + 2, + [ + "inference", + "set", + "--no-verify", + "--provider", + "compatible-endpoint", + "--model", + model, + "--timeout", + "180", + ], + { ignoreError: true }, + ); + }); + it.each([ ["HTTPS loopback", "compatible-endpoint", "https://localhost:8000/v1"], ["non-loopback host", "compatible-endpoint", "http://10.0.0.1:8000/v1"], ["missing explicit port", "compatible-endpoint", "http://localhost/v1"], ["privileged port", "compatible-endpoint", "http://localhost:1023/v1"], + ["IPv6 zone ID", "compatible-endpoint", "http://[::1%25eth0]:8000/v1"], ["embedded URL credentials", "compatible-endpoint", "http://user:pass@localhost:8000/v1"], ["malformed URL", "compatible-endpoint", "not a url"], ["non-compatible provider", "openai-api", "http://localhost:8000/v1"], @@ -182,6 +275,7 @@ describe("setupRemoteProviderInference", () => { [ "inference", "set", + ...(provider === "compatible-endpoint" ? ["--no-verify"] : []), "--provider", provider, "--model", diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index 9e7bc85f44..92106612ef 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -20,6 +20,7 @@ function gatewayReachableCompatibleEndpointUrl( return endpointUrl; } const hostname = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase(); + if (hostname.includes("%")) return endpointUrl; const port = Number(parsed.port); const isLoopback = hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1"; if ( @@ -173,6 +174,8 @@ export async function setupRemoteProviderInference( // The host-side probe validates the user-provided loopback URL. When the // provider route is rewritten to host.openshell.internal, OpenShell's // host-side verifier cannot resolve that sandbox-only bridge name. + // See ensureLocalProviderReachable in local-inference-topology.ts for the + // local-provider counterpart and its removal condition. argsv.push("--no-verify"); } argsv.push("--provider", provider, "--model", model); From 9506281950235a7a30628400ee7195f75e20c332 Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Tue, 7 Jul 2026 01:18:32 +0800 Subject: [PATCH 4/9] test(sandbox): accept rebuild notice in recovery fixture Signed-off-by: Chengjie Wang --- .../sandbox/rebuild-prepared-recovery.test.ts | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts b/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts index 3f0935d67d..5a751f2802 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts @@ -1,29 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createRequire } from "node:module"; - -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createRebuildFlowHarness, makePreparedRecoveryManifest, - snapshotEnv, + resetRebuildFlowTestEnvironment, + restoreRebuildFlowTestEnvironment, } from "../../../../test/helpers/rebuild-flow-harness"; -const requireDist = createRequire(import.meta.url); -const rebuildModulePath = "./rebuild.js"; -const restoreSandboxEnv = snapshotEnv(["NEMOCLAW_SANDBOX_NAME"]); - describe("prepared rebuild recovery", () => { - beforeEach(() => { - delete process.env.NEMOCLAW_SANDBOX_NAME; - }); - - afterEach(() => { - vi.restoreAllMocks(); - delete require.cache[requireDist.resolve(rebuildModulePath)]; - restoreSandboxEnv(); - }); + beforeEach(resetRebuildFlowTestEnvironment); + afterEach(restoreRebuildFlowTestEnvironment); it("restores the validated pre-upgrade manifest without taking a second backup (#6114)", async () => { const harness = createRebuildFlowHarness({ From 3f5adf7e522478bd263a8038781a1c75b8750b14 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 10 Jul 2026 17:58:49 +0700 Subject: [PATCH 5/9] test(inference): update self-hosted docs expectations Signed-off-by: San Dang --- test/inference-options-docs.test.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index ab056e0016..062cf79a94 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -209,11 +209,14 @@ describe("inference setup navigation", () => { expect(section).toContain("[Use Ollama for Local Inference](use-local-inference)"); }); - it("uses a loopback-only bind for the raw model server example", () => { + it("uses a container-reachable bind for the raw model server example (#5744)", () => { const markdown = fs.readFileSync(selfHostedInferenceSetupPath, "utf8"); - expect(markdown).toContain("--host 127.0.0.1"); - expect(markdown).not.toContain("--host 0.0.0.0"); + expect(markdown).toContain("--host 0.0.0.0"); + expect(markdown).not.toContain("--host 127.0.0.1"); + expect(markdown).toContain( + "a server bound only to `127.0.0.1` can still be unreachable from the sandbox route.", + ); }); it("routes vLLM tool-calling remediation to the self-hosted server guide", () => { @@ -279,10 +282,13 @@ describe("inference setup navigation", () => { expect(result.note).toContain("validation skipped"); expect(markdown).toContain("`http://host.openshell.internal:8000/v1`"); expect(markdown).toContain( - "This is a sandbox-internal alias, so host-side endpoint probing is skipped during onboarding.", + "For loopback HTTP endpoints such as `http://localhost:8000/v1`, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` so sandbox traffic can leave the container namespace.", + ); + expect(markdown).toContain( + "If you manually enter a sandbox-internal alias such as `http://host.openshell.internal:8000/v1`, host-side endpoint probing is skipped during onboarding.", ); expect(markdown).toContain( - "Use a routable endpoint when you need onboarding to verify the API, tool-calling, and streaming paths.", + "Use a host-routable endpoint such as `localhost` when you need onboarding to verify the API, tool-calling, and streaming paths", ); const textAfterEachAlias = markdown.split(hostGatewayAlias).slice(1); expect(textAfterEachAlias).toHaveLength(2); From da8f481f360c42acd60acd83d03cf9d702f1f024 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 10 Jul 2026 18:19:45 +0700 Subject: [PATCH 6/9] test(inference): cover compatible endpoint bridge boundaries Signed-off-by: San Dang --- .../local-compatible-inference-setup.mdx | 6 ++- .../inference-providers/remote.test.ts | 44 ++++++++++++++++++- src/lib/onboard/inference-providers/remote.ts | 16 +++++++ test/inference-options-docs.test.ts | 8 +++- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/docs/inference/local-compatible-inference-setup.mdx b/docs/inference/local-compatible-inference-setup.mdx index 9c05547ff9..67e4543436 100644 --- a/docs/inference/local-compatible-inference-setup.mdx +++ b/docs/inference/local-compatible-inference-setup.mdx @@ -56,7 +56,8 @@ $$nemoclaw onboard When the wizard asks you to choose an inference provider, select **Other OpenAI-compatible endpoint**. Enter the base URL of your local server, for example `http://localhost:8000/v1` on the default Docker-driver topology. -For loopback HTTP endpoints such as `http://localhost:8000/v1`, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` so sandbox traffic can leave the container namespace. +For HTTP loopback endpoints on the default HTTP port or an unprivileged port, such as `http://localhost:8000/v1`, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` so sandbox traffic can leave the container namespace. +This rewrite depends on an OpenShell topology that resolves `host.openshell.internal` inside the sandbox; if that bridge is unavailable, onboarding can still validate the host URL, but `$$nemoclaw status` is the authoritative runtime check. Make sure the local server listens on an address reachable from containers, such as `0.0.0.0`; a server bound only to `127.0.0.1` can still be unreachable from the sandbox route. Port `8000` is included in NemoClaw's `local-inference` policy preset. If you manually enter a sandbox-internal alias such as `http://host.openshell.internal:8000/v1`, host-side endpoint probing is skipped during onboarding. @@ -84,7 +85,8 @@ llama-server \ Select **Other OpenAI-compatible endpoint** during onboarding. Enter the server base URL, such as `http://localhost:8000/v1`, and the model id reported by `/v1/models`. -NemoClaw validates loopback HTTP endpoints from the host and registers the OpenShell gateway route through `host.openshell.internal:` for sandbox traffic. +For HTTP loopback endpoints on the default HTTP port or an unprivileged port, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` for sandbox traffic. +This rewrite depends on an OpenShell topology that resolves `host.openshell.internal` inside the sandbox; if that bridge is unavailable, onboarding can still validate the host URL, but `$$nemoclaw status` is the authoritative runtime check. If you manually enter a sandbox-internal alias such as `http://host.openshell.internal:8000/v1`, host-side endpoint probing is skipped during onboarding. Use a host-routable endpoint such as `localhost` when you need onboarding to verify the API, tool-calling, and streaming paths. diff --git a/src/lib/onboard/inference-providers/remote.test.ts b/src/lib/onboard/inference-providers/remote.test.ts index c5dcd59546..c4f329f534 100644 --- a/src/lib/onboard/inference-providers/remote.test.ts +++ b/src/lib/onboard/inference-providers/remote.test.ts @@ -63,6 +63,12 @@ function makeDeps(overrides: Partial = {}): RemoteProviderDe }; } +function makeDepsWithCompatibleSkipVerify(skipVerify: boolean): RemoteProviderDeps { + const deps = makeDeps(); + deps.REMOTE_PROVIDER_CONFIG.custom.skipVerify = skipVerify; + return deps; +} + describe("setupRemoteProviderInference", () => { const model = "Qwen/Qwen2.5-1.5B-Instruct"; @@ -244,10 +250,46 @@ describe("setupRemoteProviderInference", () => { ); }); + it.each([ + ["loopback route rewrite", "http://localhost:8000/v1", true], + ["non-loopback route", "http://10.0.0.1:8000/v1", false], + ])("uses --no-verify for %s only when the gateway URL differs (#5744)", async (_name, endpointUrl, expectedNoVerify) => { + const deps = makeDepsWithCompatibleSkipVerify(false); + + await expect( + setupRemoteProviderInference( + { + sandboxName: "dcode-vllm-local", + model, + provider: "compatible-endpoint", + endpointUrl, + credentialEnv: "COMPATIBLE_API_KEY", + }, + deps, + ), + ).resolves.toEqual({ done: false }); + + expect(deps.runOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + ...(expectedNoVerify ? ["--no-verify"] : []), + "--provider", + "compatible-endpoint", + "--model", + model, + "--timeout", + "180", + ], + { ignoreError: true }, + ); + }); + it.each([ ["HTTPS loopback", "compatible-endpoint", "https://localhost:8000/v1"], ["non-loopback host", "compatible-endpoint", "http://10.0.0.1:8000/v1"], - ["privileged port", "compatible-endpoint", "http://localhost:1023/v1"], + ["IPv4 privileged port", "compatible-endpoint", "http://localhost:1023/v1"], + ["IPv6 privileged port", "compatible-endpoint", "http://[::1]:1023/v1"], ["IPv6 zone ID", "compatible-endpoint", "http://[::1%25eth0]:8000/v1"], ["embedded URL credentials", "compatible-endpoint", "http://user:pass@localhost:8000/v1"], ["malformed URL", "compatible-endpoint", "not a url"], diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index e4a448d6e4..d95cc5c53a 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -22,6 +22,20 @@ const { probeOpenAiLikeEndpoint } = require("../../inference/onboard-probes") as type StaleProviderReplaceResult = { ok: boolean; status?: number | null; message?: string }; +// #5744: compatible endpoints entered as host loopback URLs must be split +// across two routing contexts. The onboard smoke probe runs on the host, so it +// should keep the user-entered URL (`http://localhost:/v1`). The sandbox +// runtime reaches the same host service through OpenShell's sandbox host bridge, +// so the provider route uses `host.openshell.internal:` instead. +// +// Source boundary: OpenShell owns the gateway bridge and provider verification +// path. NemoClaw cannot infer every deployed gateway topology from provider +// config alone, so this helper applies only to the narrow, host-validated HTTP +// loopback shape that OpenShell exposes through the sandbox host bridge. +// +// Removal condition: drop this rewrite and the paired --no-verify once +// OpenShell verifies provider routes from the gateway/sandbox routing context, +// or exposes a definitive sandbox-path provider probe for this bridge. function gatewayReachableCompatibleEndpointUrl( provider: string, endpointUrl: string | null | undefined, @@ -31,6 +45,8 @@ function gatewayReachableCompatibleEndpointUrl( try { parsed = new URL(endpointUrl); } catch { + // Leave malformed input untouched so the normal provider/probe validation + // path can report the URL problem instead of this classifier throwing first. return endpointUrl; } const hostname = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase(); diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 062cf79a94..73c46ff161 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -217,6 +217,9 @@ describe("inference setup navigation", () => { expect(markdown).toContain( "a server bound only to `127.0.0.1` can still be unreachable from the sandbox route.", ); + expect(markdown).toContain( + "This rewrite depends on an OpenShell topology that resolves `host.openshell.internal` inside the sandbox", + ); }); it("routes vLLM tool-calling remediation to the self-hosted server guide", () => { @@ -282,7 +285,10 @@ describe("inference setup navigation", () => { expect(result.note).toContain("validation skipped"); expect(markdown).toContain("`http://host.openshell.internal:8000/v1`"); expect(markdown).toContain( - "For loopback HTTP endpoints such as `http://localhost:8000/v1`, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` so sandbox traffic can leave the container namespace.", + "For HTTP loopback endpoints on the default HTTP port or an unprivileged port, such as `http://localhost:8000/v1`, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` so sandbox traffic can leave the container namespace.", + ); + expect(markdown).toContain( + "if that bridge is unavailable, onboarding can still validate the host URL, but `$$nemoclaw status` is the authoritative runtime check.", ); expect(markdown).toContain( "If you manually enter a sandbox-internal alias such as `http://host.openshell.internal:8000/v1`, host-side endpoint probing is skipped during onboarding.", From 3cbe47efdd59600fdddbc001457a27095ea5e8e5 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 10 Jul 2026 18:40:37 +0700 Subject: [PATCH 7/9] test(ci): relax subprocess shard timing Signed-off-by: San Dang --- .../inference-set-compatible-provider.test.ts | 6 +- .../mcp-bridge-status-resolution.test.ts | 452 ++++++++++-------- 2 files changed, 257 insertions(+), 201 deletions(-) diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index 31a69c0688..cf3d74b292 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; +import { rewriteConfigUrlsWithDnsPinning } from "../sandbox/config"; import type { ConfigObject } from "../security/credential-filter"; import { runInferenceSet } from "./inference-set"; import { baseSession, createDeps } from "./inference-set.test-support"; @@ -304,8 +305,6 @@ describe("runInferenceSet compatible providers", () => { ["privileged-port bridge", "http://host.openshell.internal:80/v1", "93.184.216.34"], ["DNS-private", "https://private-resolution.example/v1", "10.0.0.8"], ])(`rejects %s endpoint metadata for ${provider}`, async (_kind, endpointUrl, resolvedAddress) => { - const actualConfig = - await vi.importActual("../sandbox/config"); const lookup = vi.fn(async () => [{ address: resolvedAddress, family: 4 }]); const deps = createDeps({ config: { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } } }, @@ -315,8 +314,7 @@ describe("runInferenceSet compatible providers", () => { provider: "nvidia-prod", model: "nvidia/model-a", }, - rewriteConfigUrlsWithDnsPinning: (value) => - actualConfig.rewriteConfigUrlsWithDnsPinning(value, lookup), + rewriteConfigUrlsWithDnsPinning: (value) => rewriteConfigUrlsWithDnsPinning(value, lookup), }); await expect( diff --git a/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts b/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts index aba649976e..6be873804c 100644 --- a/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts @@ -13,6 +13,8 @@ const sourceNodeOptions = [process.env.NODE_OPTIONS, `--require=${sourceRequireH .filter(Boolean) .join(" "); const tempHomes = new Set(); +const HARNESS_TIMEOUT_MS = 15_000; +const SUBPROCESS_TEST_TIMEOUT_MS = HARNESS_TIMEOUT_MS + 5_000; function createTempHome(prefix: string): string { const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -141,17 +143,28 @@ ${body} cwd: process.cwd(), encoding: "utf8", env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, + timeout: HARNESS_TIMEOUT_MS, }); - expect(result.status, `harness failed: ${result.stderr}`).toBe(0); + const failureDetail = [ + result.error ? `error: ${result.error.message}` : "", + result.signal ? `signal: ${result.signal}` : "", + result.stderr ? `stderr: ${result.stderr}` : "", + result.stdout ? `stdout: ${result.stdout}` : "", + ] + .filter(Boolean) + .join("\n"); + expect(result.status, `harness failed: ${failureDetail}`).toBe(0); return { status: result.status, stdout: result.stdout }; } describe("MCP status wire-level credential-resolution probe", () => { - it("probes by default for a single named server and surfaces the wire failure (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-single-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "probes by default for a single named server and surfaces the wire failure (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-single-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--json"]); const status = JSON.parse(logLines.join("\n")); process.stdout.write(JSON.stringify({ @@ -160,34 +173,38 @@ describe("MCP status wire-level credential-resolution probe", () => { exitCode: process.exitCode ?? 0, })); `, - ); - const payload = JSON.parse(stdout) as { - status: { - provider: { credentialResolution?: { ok: boolean | null; httpStatus?: number } }; - warnings: string[]; + ); + const payload = JSON.parse(stdout) as { + status: { + provider: { credentialResolution?: { ok: boolean | null; httpStatus?: number } }; + warnings: string[]; + }; + probed: boolean; + exitCode: number; }; - probed: boolean; - exitCode: number; - }; - expect(payload.probed).toBe(true); - expect(payload.status.provider.credentialResolution).toMatchObject({ - ok: null, - httpStatus: 401, - controlHttpStatus: 401, - }); - expect( - payload.status.warnings.some((warning) => - warning.includes("Credential resolution could not be verified"), - ), - ).toBe(true); - expect(payload.exitCode).toBe(0); - }); + expect(payload.probed).toBe(true); + expect(payload.status.provider.credentialResolution).toMatchObject({ + ok: null, + httpStatus: 401, + controlHttpStatus: 401, + }); + expect( + payload.status.warnings.some((warning) => + warning.includes("Credential resolution could not be verified"), + ), + ).toBe(true); + expect(payload.exitCode).toBe(0); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("skips status probe traffic until exact policy and provider readiness are verified (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-readiness-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "skips status probe traffic until exact policy and provider readiness are verified (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-readiness-"); + const { stdout } = runHarness( + home, + String.raw` const outcomes = []; for (const policyState of ["absent", "drift", null]) { activePolicyState = policyState; @@ -231,70 +248,82 @@ describe("MCP status wire-level credential-resolution probe", () => { }); process.stdout.write(JSON.stringify(outcomes)); `, - ); - const outcomes = JSON.parse(stdout) as Array<{ - case: string; - gatewayPresent?: boolean | null; - resolution: { ok: boolean | null; detail?: string }; - probed: boolean; - }>; - expect(outcomes).toHaveLength(6); - expect(outcomes.map((outcome) => outcome.gatewayPresent).slice(0, 3)).toEqual([ - false, - false, - null, - ]); - for (const outcome of outcomes) { - expect(outcome.probed, outcome.case).toBe(false); - expect(outcome.resolution.ok, outcome.case).toBeNull(); - expect(outcome.resolution.detail, outcome.case).toContain("probe skipped"); - } - }); + ); + const outcomes = JSON.parse(stdout) as Array<{ + case: string; + gatewayPresent?: boolean | null; + resolution: { ok: boolean | null; detail?: string }; + probed: boolean; + }>; + expect(outcomes).toHaveLength(6); + expect(outcomes.map((outcome) => outcome.gatewayPresent).slice(0, 3)).toEqual([ + false, + false, + null, + ]); + for (const outcome of outcomes) { + expect(outcome.probed, outcome.case).toBe(false); + expect(outcome.resolution.ok, outcome.case).toBeNull(); + expect(outcome.resolution.detail, outcome.case).toContain("probe skipped"); + } + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("renders the identical-rejection probe in the human-readable status output (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-render-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "renders the identical-rejection probe in the human-readable status output (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-render-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github"]); process.stdout.write(JSON.stringify({ lines: logLines })); `, - ); - const payload = JSON.parse(stdout) as { lines: string[] }; - expect(payload.lines.some((line) => line.includes("credential resolution: unknown"))).toBe( - true, - ); - expect( - payload.lines.some((line) => line.includes("Credential resolution could not be verified")), - ).toBe(true); - }); + ); + const payload = JSON.parse(stdout) as { lines: string[] }; + expect(payload.lines.some((line) => line.includes("credential resolution: unknown"))).toBe( + true, + ); + expect( + payload.lines.some((line) => line.includes("Credential resolution could not be verified")), + ).toBe(true); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("keeps the status warning for identical 400 explicitly inconclusive (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-400-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "keeps the status warning for identical 400 explicitly inconclusive (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-400-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--json"]); const status = JSON.parse(logLines.join("\n")); process.stdout.write(JSON.stringify({ warnings: status.warnings })); `, - { probeHttpStatus: 400 }, - ); - const payload = JSON.parse(stdout) as { warnings: string[] }; - const warning = payload.warnings.find((line) => - line.includes("Credential resolution could not be verified"), - ); - expect(warning).toBeDefined(); - expect(warning).toContain("inconclusive even with a valid stored credential"); - expect(warning).toContain("request validation"); - expect(warning).not.toContain("the OpenShell host is not rewriting"); - }); + { probeHttpStatus: 400 }, + ); + const payload = JSON.parse(stdout) as { warnings: string[] }; + const warning = payload.warnings.find((line) => + line.includes("Credential resolution could not be verified"), + ); + expect(warning).toBeDefined(); + expect(warning).toContain("inconclusive even with a valid stored credential"); + expect(warning).toContain("request validation"); + expect(warning).not.toContain("the OpenShell host is not rewriting"); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("never probes from bare status or list so multi-server views stay fast (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-list-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "never probes from bare status or list so multi-server views stay fast (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-list-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "--json"]); const bareStatus = JSON.parse(logLines.join("\n")); logLines.length = 0; @@ -306,22 +335,26 @@ describe("MCP status wire-level credential-resolution probe", () => { listResolution: list.bridges[0].provider.credentialResolution ?? null, })); `, - ); - const payload = JSON.parse(stdout) as { - probed: boolean; - bareStatusResolution: unknown; - listResolution: unknown; - }; - expect(payload.probed).toBe(false); - expect(payload.bareStatusResolution).toBeNull(); - expect(payload.listResolution).toBeNull(); - }); + ); + const payload = JSON.parse(stdout) as { + probed: boolean; + bareStatusResolution: unknown; + listResolution: unknown; + }; + expect(payload.probed).toBe(false); + expect(payload.bareStatusResolution).toBeNull(); + expect(payload.listResolution).toBeNull(); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("honors --no-probe on a named server and --probe on the multi-server form (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-flags-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "honors --no-probe on a named server and --probe on the multi-server form (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-flags-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--no-probe", "--json"]); const skipped = JSON.parse(logLines.join("\n")); const probesAfterSkip = executedSandboxCommands.filter((c) => c.includes("NEMOCLAW_MCP_PROBE")).length; @@ -334,44 +367,52 @@ describe("MCP status wire-level credential-resolution probe", () => { forcedResolution: forced.bridges[0].provider.credentialResolution ?? null, })); `, - ); - const payload = JSON.parse(stdout) as { - probesAfterSkip: number; - skippedResolution: unknown; - forcedResolution: { ok: boolean | null; httpStatus?: number } | null; - }; - expect(payload.probesAfterSkip).toBe(0); - expect(payload.skippedResolution).toBeNull(); - expect(payload.forcedResolution).toMatchObject({ - ok: null, - httpStatus: 401, - controlHttpStatus: 401, - }); - }); + ); + const payload = JSON.parse(stdout) as { + probesAfterSkip: number; + skippedResolution: unknown; + forcedResolution: { ok: boolean | null; httpStatus?: number } | null; + }; + expect(payload.probesAfterSkip).toBe(0); + expect(payload.skippedResolution).toBeNull(); + expect(payload.forcedResolution).toMatchObject({ + ok: null, + httpStatus: 401, + controlHttpStatus: 401, + }); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("rejects combining --probe with --no-probe (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-conflict-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "rejects combining --probe with --no-probe (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-conflict-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--probe", "--no-probe"]); const observedExitCode = process.exitCode ?? 0; process.exitCode = 0; process.stdout.write(JSON.stringify({ errorLines, exitCode: observedExitCode })); `, - ); - const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; - expect(payload.exitCode).toBe(2); - expect(payload.errorLines.join("\n")).toContain("at most one of --probe / --no-probe"); - }); + ); + const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; + expect(payload.exitCode).toBe(2); + expect(payload.errorLines.join("\n")).toContain("at most one of --probe / --no-probe"); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); }); describe("MCP add post-add credential-resolution probe", () => { - it("warns loudly on an identical-rejection probe without failing the committed add (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-add-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "warns loudly on an identical-rejection probe without failing the committed add (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-add-"); + const { stdout } = runHarness( + home, + String.raw` const addRestart = require("./src/lib/actions/sandbox/mcp-bridge-add-restart.js"); addRestart.addMcpBridge = async () => {}; await bridge.dispatchMcpBridgeCommand("alpha", [ @@ -384,29 +425,36 @@ describe("MCP add post-add credential-resolution probe", () => { exitCode: process.exitCode ?? 0, })); `, - ); - const payload = JSON.parse(stdout) as { - logLines: string[]; - errorLines: string[]; - probed: boolean; - exitCode: number; - }; - expect(payload.probed).toBe(true); - expect(payload.logLines.some((line) => line.includes("MCP server 'github' added"))).toBe(true); - expect( - payload.errorLines.some( - (line) => - line.includes("WARNING") && line.includes("Credential resolution could not be verified"), - ), - ).toBe(true); - expect(payload.exitCode).toBe(0); - }); + ); + const payload = JSON.parse(stdout) as { + logLines: string[]; + errorLines: string[]; + probed: boolean; + exitCode: number; + }; + expect(payload.probed).toBe(true); + expect(payload.logLines.some((line) => line.includes("MCP server 'github' added"))).toBe( + true, + ); + expect( + payload.errorLines.some( + (line) => + line.includes("WARNING") && + line.includes("Credential resolution could not be verified"), + ), + ).toBe(true); + expect(payload.exitCode).toBe(0); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("skips post-add probe traffic when policy verification is absent, drifted, or unknown (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-add-policy-gate-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "skips post-add probe traffic when policy verification is absent, drifted, or unknown (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-add-policy-gate-"); + const { stdout } = runHarness( + home, + String.raw` const addRestart = require("./src/lib/actions/sandbox/mcp-bridge-add-restart.js"); addRestart.addMcpBridge = async () => {}; const outcomes = []; @@ -427,27 +475,31 @@ describe("MCP add post-add credential-resolution probe", () => { } process.stdout.write(JSON.stringify(outcomes)); `, - ); - const outcomes = JSON.parse(stdout) as Array<{ - policyState: "absent" | "drift" | null; - probed: boolean; - output: string; - exitCode: number; - }>; - expect(outcomes).toHaveLength(3); - for (const outcome of outcomes) { - expect(outcome.probed, String(outcome.policyState)).toBe(false); - expect(outcome.output).toContain("Credential resolution probe was inconclusive"); - expect(outcome.output).toContain("probe skipped"); - expect(outcome.exitCode).toBe(0); - } - }); + ); + const outcomes = JSON.parse(stdout) as Array<{ + policyState: "absent" | "drift" | null; + probed: boolean; + output: string; + exitCode: number; + }>; + expect(outcomes).toHaveLength(3); + for (const outcome of outcomes) { + expect(outcome.probed, String(outcome.policyState)).toBe(false); + expect(outcome.output).toContain("Credential resolution probe was inconclusive"); + expect(outcome.output).toContain("probe skipped"); + expect(outcome.exitCode).toBe(0); + } + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("keeps the post-add warning for identical 400 explicitly inconclusive (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-add-400-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "keeps the post-add warning for identical 400 explicitly inconclusive (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-add-400-"); + const { stdout } = runHarness( + home, + String.raw` const addRestart = require("./src/lib/actions/sandbox/mcp-bridge-add-restart.js"); addRestart.addMcpBridge = async () => {}; await bridge.dispatchMcpBridgeCommand("alpha", [ @@ -455,22 +507,26 @@ describe("MCP add post-add credential-resolution probe", () => { ]); process.stdout.write(JSON.stringify({ errorLines, exitCode: process.exitCode ?? 0 })); `, - { probeHttpStatus: 400 }, - ); - const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; - const warning = payload.errorLines.find((line) => line.includes("WARNING")); - expect(warning).toBeDefined(); - expect(warning).toContain("inconclusive even with a valid stored credential"); - expect(warning).toContain("request validation"); - expect(warning).not.toContain("the OpenShell host is not rewriting"); - expect(payload.exitCode).toBe(0); - }); + { probeHttpStatus: 400 }, + ); + const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; + const warning = payload.errorLines.find((line) => line.includes("WARNING")); + expect(warning).toBeDefined(); + expect(warning).toContain("inconclusive even with a valid stored credential"); + expect(warning).toContain("request validation"); + expect(warning).not.toContain("the OpenShell host is not rewriting"); + expect(payload.exitCode).toBe(0); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); - it("skips the post-add probe when --no-probe is passed (#6379)", () => { - const home = createTempHome("nemoclaw-mcp-resolution-add-skip-"); - const { stdout } = runHarness( - home, - String.raw` + it( + "skips the post-add probe when --no-probe is passed (#6379)", + () => { + const home = createTempHome("nemoclaw-mcp-resolution-add-skip-"); + const { stdout } = runHarness( + home, + String.raw` const addRestart = require("./src/lib/actions/sandbox/mcp-bridge-add-restart.js"); addRestart.addMcpBridge = async () => {}; await bridge.dispatchMcpBridgeCommand("alpha", [ @@ -482,14 +538,16 @@ describe("MCP add post-add credential-resolution probe", () => { exitCode: process.exitCode ?? 0, })); `, - ); - const payload = JSON.parse(stdout) as { - errorLines: string[]; - probed: boolean; - exitCode: number; - }; - expect(payload.probed).toBe(false); - expect(payload.errorLines).toHaveLength(0); - expect(payload.exitCode).toBe(0); - }); + ); + const payload = JSON.parse(stdout) as { + errorLines: string[]; + probed: boolean; + exitCode: number; + }; + expect(payload.probed).toBe(false); + expect(payload.errorLines).toHaveLength(0); + expect(payload.exitCode).toBe(0); + }, + SUBPROCESS_TEST_TIMEOUT_MS, + ); }); From a4c1eb46c08a191ee9502c243f75d0572fcd2462 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 10 Jul 2026 18:43:44 +0700 Subject: [PATCH 8/9] Revert "test(ci): relax subprocess shard timing" This reverts commit 3cbe47efdd59600fdddbc001457a27095ea5e8e5. Signed-off-by: San Dang --- .../inference-set-compatible-provider.test.ts | 6 +- .../mcp-bridge-status-resolution.test.ts | 452 ++++++++---------- 2 files changed, 201 insertions(+), 257 deletions(-) diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index cf3d74b292..31a69c0688 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; -import { rewriteConfigUrlsWithDnsPinning } from "../sandbox/config"; import type { ConfigObject } from "../security/credential-filter"; import { runInferenceSet } from "./inference-set"; import { baseSession, createDeps } from "./inference-set.test-support"; @@ -305,6 +304,8 @@ describe("runInferenceSet compatible providers", () => { ["privileged-port bridge", "http://host.openshell.internal:80/v1", "93.184.216.34"], ["DNS-private", "https://private-resolution.example/v1", "10.0.0.8"], ])(`rejects %s endpoint metadata for ${provider}`, async (_kind, endpointUrl, resolvedAddress) => { + const actualConfig = + await vi.importActual("../sandbox/config"); const lookup = vi.fn(async () => [{ address: resolvedAddress, family: 4 }]); const deps = createDeps({ config: { agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } } }, @@ -314,7 +315,8 @@ describe("runInferenceSet compatible providers", () => { provider: "nvidia-prod", model: "nvidia/model-a", }, - rewriteConfigUrlsWithDnsPinning: (value) => rewriteConfigUrlsWithDnsPinning(value, lookup), + rewriteConfigUrlsWithDnsPinning: (value) => + actualConfig.rewriteConfigUrlsWithDnsPinning(value, lookup), }); await expect( diff --git a/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts b/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts index 6be873804c..aba649976e 100644 --- a/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts @@ -13,8 +13,6 @@ const sourceNodeOptions = [process.env.NODE_OPTIONS, `--require=${sourceRequireH .filter(Boolean) .join(" "); const tempHomes = new Set(); -const HARNESS_TIMEOUT_MS = 15_000; -const SUBPROCESS_TEST_TIMEOUT_MS = HARNESS_TIMEOUT_MS + 5_000; function createTempHome(prefix: string): string { const home = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -143,28 +141,17 @@ ${body} cwd: process.cwd(), encoding: "utf8", env: { ...process.env, HOME: home, NODE_OPTIONS: sourceNodeOptions }, - timeout: HARNESS_TIMEOUT_MS, }); - const failureDetail = [ - result.error ? `error: ${result.error.message}` : "", - result.signal ? `signal: ${result.signal}` : "", - result.stderr ? `stderr: ${result.stderr}` : "", - result.stdout ? `stdout: ${result.stdout}` : "", - ] - .filter(Boolean) - .join("\n"); - expect(result.status, `harness failed: ${failureDetail}`).toBe(0); + expect(result.status, `harness failed: ${result.stderr}`).toBe(0); return { status: result.status, stdout: result.stdout }; } describe("MCP status wire-level credential-resolution probe", () => { - it( - "probes by default for a single named server and surfaces the wire failure (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-single-"); - const { stdout } = runHarness( - home, - String.raw` + it("probes by default for a single named server and surfaces the wire failure (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-single-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--json"]); const status = JSON.parse(logLines.join("\n")); process.stdout.write(JSON.stringify({ @@ -173,38 +160,34 @@ describe("MCP status wire-level credential-resolution probe", () => { exitCode: process.exitCode ?? 0, })); `, - ); - const payload = JSON.parse(stdout) as { - status: { - provider: { credentialResolution?: { ok: boolean | null; httpStatus?: number } }; - warnings: string[]; - }; - probed: boolean; - exitCode: number; + ); + const payload = JSON.parse(stdout) as { + status: { + provider: { credentialResolution?: { ok: boolean | null; httpStatus?: number } }; + warnings: string[]; }; - expect(payload.probed).toBe(true); - expect(payload.status.provider.credentialResolution).toMatchObject({ - ok: null, - httpStatus: 401, - controlHttpStatus: 401, - }); - expect( - payload.status.warnings.some((warning) => - warning.includes("Credential resolution could not be verified"), - ), - ).toBe(true); - expect(payload.exitCode).toBe(0); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + probed: boolean; + exitCode: number; + }; + expect(payload.probed).toBe(true); + expect(payload.status.provider.credentialResolution).toMatchObject({ + ok: null, + httpStatus: 401, + controlHttpStatus: 401, + }); + expect( + payload.status.warnings.some((warning) => + warning.includes("Credential resolution could not be verified"), + ), + ).toBe(true); + expect(payload.exitCode).toBe(0); + }); - it( - "skips status probe traffic until exact policy and provider readiness are verified (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-readiness-"); - const { stdout } = runHarness( - home, - String.raw` + it("skips status probe traffic until exact policy and provider readiness are verified (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-readiness-"); + const { stdout } = runHarness( + home, + String.raw` const outcomes = []; for (const policyState of ["absent", "drift", null]) { activePolicyState = policyState; @@ -248,82 +231,70 @@ describe("MCP status wire-level credential-resolution probe", () => { }); process.stdout.write(JSON.stringify(outcomes)); `, - ); - const outcomes = JSON.parse(stdout) as Array<{ - case: string; - gatewayPresent?: boolean | null; - resolution: { ok: boolean | null; detail?: string }; - probed: boolean; - }>; - expect(outcomes).toHaveLength(6); - expect(outcomes.map((outcome) => outcome.gatewayPresent).slice(0, 3)).toEqual([ - false, - false, - null, - ]); - for (const outcome of outcomes) { - expect(outcome.probed, outcome.case).toBe(false); - expect(outcome.resolution.ok, outcome.case).toBeNull(); - expect(outcome.resolution.detail, outcome.case).toContain("probe skipped"); - } - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + ); + const outcomes = JSON.parse(stdout) as Array<{ + case: string; + gatewayPresent?: boolean | null; + resolution: { ok: boolean | null; detail?: string }; + probed: boolean; + }>; + expect(outcomes).toHaveLength(6); + expect(outcomes.map((outcome) => outcome.gatewayPresent).slice(0, 3)).toEqual([ + false, + false, + null, + ]); + for (const outcome of outcomes) { + expect(outcome.probed, outcome.case).toBe(false); + expect(outcome.resolution.ok, outcome.case).toBeNull(); + expect(outcome.resolution.detail, outcome.case).toContain("probe skipped"); + } + }); - it( - "renders the identical-rejection probe in the human-readable status output (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-render-"); - const { stdout } = runHarness( - home, - String.raw` + it("renders the identical-rejection probe in the human-readable status output (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-render-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github"]); process.stdout.write(JSON.stringify({ lines: logLines })); `, - ); - const payload = JSON.parse(stdout) as { lines: string[] }; - expect(payload.lines.some((line) => line.includes("credential resolution: unknown"))).toBe( - true, - ); - expect( - payload.lines.some((line) => line.includes("Credential resolution could not be verified")), - ).toBe(true); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + ); + const payload = JSON.parse(stdout) as { lines: string[] }; + expect(payload.lines.some((line) => line.includes("credential resolution: unknown"))).toBe( + true, + ); + expect( + payload.lines.some((line) => line.includes("Credential resolution could not be verified")), + ).toBe(true); + }); - it( - "keeps the status warning for identical 400 explicitly inconclusive (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-400-"); - const { stdout } = runHarness( - home, - String.raw` + it("keeps the status warning for identical 400 explicitly inconclusive (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-400-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--json"]); const status = JSON.parse(logLines.join("\n")); process.stdout.write(JSON.stringify({ warnings: status.warnings })); `, - { probeHttpStatus: 400 }, - ); - const payload = JSON.parse(stdout) as { warnings: string[] }; - const warning = payload.warnings.find((line) => - line.includes("Credential resolution could not be verified"), - ); - expect(warning).toBeDefined(); - expect(warning).toContain("inconclusive even with a valid stored credential"); - expect(warning).toContain("request validation"); - expect(warning).not.toContain("the OpenShell host is not rewriting"); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + { probeHttpStatus: 400 }, + ); + const payload = JSON.parse(stdout) as { warnings: string[] }; + const warning = payload.warnings.find((line) => + line.includes("Credential resolution could not be verified"), + ); + expect(warning).toBeDefined(); + expect(warning).toContain("inconclusive even with a valid stored credential"); + expect(warning).toContain("request validation"); + expect(warning).not.toContain("the OpenShell host is not rewriting"); + }); - it( - "never probes from bare status or list so multi-server views stay fast (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-list-"); - const { stdout } = runHarness( - home, - String.raw` + it("never probes from bare status or list so multi-server views stay fast (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-list-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "--json"]); const bareStatus = JSON.parse(logLines.join("\n")); logLines.length = 0; @@ -335,26 +306,22 @@ describe("MCP status wire-level credential-resolution probe", () => { listResolution: list.bridges[0].provider.credentialResolution ?? null, })); `, - ); - const payload = JSON.parse(stdout) as { - probed: boolean; - bareStatusResolution: unknown; - listResolution: unknown; - }; - expect(payload.probed).toBe(false); - expect(payload.bareStatusResolution).toBeNull(); - expect(payload.listResolution).toBeNull(); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + ); + const payload = JSON.parse(stdout) as { + probed: boolean; + bareStatusResolution: unknown; + listResolution: unknown; + }; + expect(payload.probed).toBe(false); + expect(payload.bareStatusResolution).toBeNull(); + expect(payload.listResolution).toBeNull(); + }); - it( - "honors --no-probe on a named server and --probe on the multi-server form (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-flags-"); - const { stdout } = runHarness( - home, - String.raw` + it("honors --no-probe on a named server and --probe on the multi-server form (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-flags-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--no-probe", "--json"]); const skipped = JSON.parse(logLines.join("\n")); const probesAfterSkip = executedSandboxCommands.filter((c) => c.includes("NEMOCLAW_MCP_PROBE")).length; @@ -367,52 +334,44 @@ describe("MCP status wire-level credential-resolution probe", () => { forcedResolution: forced.bridges[0].provider.credentialResolution ?? null, })); `, - ); - const payload = JSON.parse(stdout) as { - probesAfterSkip: number; - skippedResolution: unknown; - forcedResolution: { ok: boolean | null; httpStatus?: number } | null; - }; - expect(payload.probesAfterSkip).toBe(0); - expect(payload.skippedResolution).toBeNull(); - expect(payload.forcedResolution).toMatchObject({ - ok: null, - httpStatus: 401, - controlHttpStatus: 401, - }); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + ); + const payload = JSON.parse(stdout) as { + probesAfterSkip: number; + skippedResolution: unknown; + forcedResolution: { ok: boolean | null; httpStatus?: number } | null; + }; + expect(payload.probesAfterSkip).toBe(0); + expect(payload.skippedResolution).toBeNull(); + expect(payload.forcedResolution).toMatchObject({ + ok: null, + httpStatus: 401, + controlHttpStatus: 401, + }); + }); - it( - "rejects combining --probe with --no-probe (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-conflict-"); - const { stdout } = runHarness( - home, - String.raw` + it("rejects combining --probe with --no-probe (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-conflict-"); + const { stdout } = runHarness( + home, + String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--probe", "--no-probe"]); const observedExitCode = process.exitCode ?? 0; process.exitCode = 0; process.stdout.write(JSON.stringify({ errorLines, exitCode: observedExitCode })); `, - ); - const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; - expect(payload.exitCode).toBe(2); - expect(payload.errorLines.join("\n")).toContain("at most one of --probe / --no-probe"); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + ); + const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; + expect(payload.exitCode).toBe(2); + expect(payload.errorLines.join("\n")).toContain("at most one of --probe / --no-probe"); + }); }); describe("MCP add post-add credential-resolution probe", () => { - it( - "warns loudly on an identical-rejection probe without failing the committed add (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-add-"); - const { stdout } = runHarness( - home, - String.raw` + it("warns loudly on an identical-rejection probe without failing the committed add (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-add-"); + const { stdout } = runHarness( + home, + String.raw` const addRestart = require("./src/lib/actions/sandbox/mcp-bridge-add-restart.js"); addRestart.addMcpBridge = async () => {}; await bridge.dispatchMcpBridgeCommand("alpha", [ @@ -425,36 +384,29 @@ describe("MCP add post-add credential-resolution probe", () => { exitCode: process.exitCode ?? 0, })); `, - ); - const payload = JSON.parse(stdout) as { - logLines: string[]; - errorLines: string[]; - probed: boolean; - exitCode: number; - }; - expect(payload.probed).toBe(true); - expect(payload.logLines.some((line) => line.includes("MCP server 'github' added"))).toBe( - true, - ); - expect( - payload.errorLines.some( - (line) => - line.includes("WARNING") && - line.includes("Credential resolution could not be verified"), - ), - ).toBe(true); - expect(payload.exitCode).toBe(0); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + ); + const payload = JSON.parse(stdout) as { + logLines: string[]; + errorLines: string[]; + probed: boolean; + exitCode: number; + }; + expect(payload.probed).toBe(true); + expect(payload.logLines.some((line) => line.includes("MCP server 'github' added"))).toBe(true); + expect( + payload.errorLines.some( + (line) => + line.includes("WARNING") && line.includes("Credential resolution could not be verified"), + ), + ).toBe(true); + expect(payload.exitCode).toBe(0); + }); - it( - "skips post-add probe traffic when policy verification is absent, drifted, or unknown (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-add-policy-gate-"); - const { stdout } = runHarness( - home, - String.raw` + it("skips post-add probe traffic when policy verification is absent, drifted, or unknown (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-add-policy-gate-"); + const { stdout } = runHarness( + home, + String.raw` const addRestart = require("./src/lib/actions/sandbox/mcp-bridge-add-restart.js"); addRestart.addMcpBridge = async () => {}; const outcomes = []; @@ -475,31 +427,27 @@ describe("MCP add post-add credential-resolution probe", () => { } process.stdout.write(JSON.stringify(outcomes)); `, - ); - const outcomes = JSON.parse(stdout) as Array<{ - policyState: "absent" | "drift" | null; - probed: boolean; - output: string; - exitCode: number; - }>; - expect(outcomes).toHaveLength(3); - for (const outcome of outcomes) { - expect(outcome.probed, String(outcome.policyState)).toBe(false); - expect(outcome.output).toContain("Credential resolution probe was inconclusive"); - expect(outcome.output).toContain("probe skipped"); - expect(outcome.exitCode).toBe(0); - } - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + ); + const outcomes = JSON.parse(stdout) as Array<{ + policyState: "absent" | "drift" | null; + probed: boolean; + output: string; + exitCode: number; + }>; + expect(outcomes).toHaveLength(3); + for (const outcome of outcomes) { + expect(outcome.probed, String(outcome.policyState)).toBe(false); + expect(outcome.output).toContain("Credential resolution probe was inconclusive"); + expect(outcome.output).toContain("probe skipped"); + expect(outcome.exitCode).toBe(0); + } + }); - it( - "keeps the post-add warning for identical 400 explicitly inconclusive (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-add-400-"); - const { stdout } = runHarness( - home, - String.raw` + it("keeps the post-add warning for identical 400 explicitly inconclusive (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-add-400-"); + const { stdout } = runHarness( + home, + String.raw` const addRestart = require("./src/lib/actions/sandbox/mcp-bridge-add-restart.js"); addRestart.addMcpBridge = async () => {}; await bridge.dispatchMcpBridgeCommand("alpha", [ @@ -507,26 +455,22 @@ describe("MCP add post-add credential-resolution probe", () => { ]); process.stdout.write(JSON.stringify({ errorLines, exitCode: process.exitCode ?? 0 })); `, - { probeHttpStatus: 400 }, - ); - const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; - const warning = payload.errorLines.find((line) => line.includes("WARNING")); - expect(warning).toBeDefined(); - expect(warning).toContain("inconclusive even with a valid stored credential"); - expect(warning).toContain("request validation"); - expect(warning).not.toContain("the OpenShell host is not rewriting"); - expect(payload.exitCode).toBe(0); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + { probeHttpStatus: 400 }, + ); + const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; + const warning = payload.errorLines.find((line) => line.includes("WARNING")); + expect(warning).toBeDefined(); + expect(warning).toContain("inconclusive even with a valid stored credential"); + expect(warning).toContain("request validation"); + expect(warning).not.toContain("the OpenShell host is not rewriting"); + expect(payload.exitCode).toBe(0); + }); - it( - "skips the post-add probe when --no-probe is passed (#6379)", - () => { - const home = createTempHome("nemoclaw-mcp-resolution-add-skip-"); - const { stdout } = runHarness( - home, - String.raw` + it("skips the post-add probe when --no-probe is passed (#6379)", () => { + const home = createTempHome("nemoclaw-mcp-resolution-add-skip-"); + const { stdout } = runHarness( + home, + String.raw` const addRestart = require("./src/lib/actions/sandbox/mcp-bridge-add-restart.js"); addRestart.addMcpBridge = async () => {}; await bridge.dispatchMcpBridgeCommand("alpha", [ @@ -538,16 +482,14 @@ describe("MCP add post-add credential-resolution probe", () => { exitCode: process.exitCode ?? 0, })); `, - ); - const payload = JSON.parse(stdout) as { - errorLines: string[]; - probed: boolean; - exitCode: number; - }; - expect(payload.probed).toBe(false); - expect(payload.errorLines).toHaveLength(0); - expect(payload.exitCode).toBe(0); - }, - SUBPROCESS_TEST_TIMEOUT_MS, - ); + ); + const payload = JSON.parse(stdout) as { + errorLines: string[]; + probed: boolean; + exitCode: number; + }; + expect(payload.probed).toBe(false); + expect(payload.errorLines).toHaveLength(0); + expect(payload.exitCode).toBe(0); + }); }); From dc666e5759ddcb495e6d143563f933626f171c23 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 10 Jul 2026 18:46:00 +0700 Subject: [PATCH 9/9] test(inference): simplify loopback gateway coverage Signed-off-by: San Dang --- .../inference-providers/remote.test.ts | 336 ------------------ src/lib/onboard/inference-providers/remote.ts | 25 +- 2 files changed, 4 insertions(+), 357 deletions(-) delete mode 100644 src/lib/onboard/inference-providers/remote.test.ts diff --git a/src/lib/onboard/inference-providers/remote.test.ts b/src/lib/onboard/inference-providers/remote.test.ts deleted file mode 100644 index c4f329f534..0000000000 --- a/src/lib/onboard/inference-providers/remote.test.ts +++ /dev/null @@ -1,336 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it, vi } from "vitest"; - -import { setupRemoteProviderInference } from "./remote"; -import type { RemoteProviderDeps } from "./types"; - -class ExitError extends Error { - constructor(readonly code: number) { - super(`EXIT_CALLED:${code}`); - } -} - -function makeDeps(overrides: Partial = {}): RemoteProviderDeps { - return { - runOpenshell: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), - upsertProvider: vi.fn(() => ({ ok: true })), - verifyInferenceRoute: vi.fn(), - verifyOnboardInferenceSmoke: vi.fn(), - isNonInteractive: vi.fn(() => true), - registry: { updateSandbox: vi.fn() as never }, - exitProcess: vi.fn((code: number): never => { - throw new ExitError(code); - }), - error: vi.fn(), - log: vi.fn(), - REMOTE_PROVIDER_CONFIG: { - custom: { - label: "Other OpenAI-compatible endpoint", - providerName: "compatible-endpoint", - providerType: "openai", - credentialEnv: "COMPATIBLE_API_KEY", - endpointUrl: "https://example.invalid/v1", - helpUrl: null, - modelMode: "input", - defaultModel: "custom-model", - // Mirrors production: compatible endpoints already skip OpenShell's - // verifier; these tests focus on which base URL gets registered. - skipVerify: true, - }, - openai: { - label: "OpenAI API", - providerName: "openai-api", - providerType: "openai", - credentialEnv: "OPENAI_API_KEY", - endpointUrl: "https://api.openai.com/v1", - helpUrl: null, - modelMode: "input", - defaultModel: "gpt-4.1", - }, - }, - hydrateCredentialEnv: vi.fn(() => "dummy-key"), - promptValidationRecovery: vi.fn(async () => "selection" as const), - classifyApplyFailure: vi.fn(() => ({ kind: "unknown" }) as never), - LOCAL_INFERENCE_TIMEOUT_SECS: 180, - bedrockRuntimeOnboard: { - setupBedrockRuntimeInference: vi.fn(async () => ({ handled: false }) as const), - }, - redact: vi.fn((input: string) => input), - compactText: vi.fn((input: string) => input.trim()), - ...overrides, - }; -} - -function makeDepsWithCompatibleSkipVerify(skipVerify: boolean): RemoteProviderDeps { - const deps = makeDeps(); - deps.REMOTE_PROVIDER_CONFIG.custom.skipVerify = skipVerify; - return deps; -} - -describe("setupRemoteProviderInference", () => { - const model = "Qwen/Qwen2.5-1.5B-Instruct"; - - it("uses a sandbox-facing host alias for loopback compatible endpoints (#5744)", async () => { - const deps = makeDeps(); - - await expect( - setupRemoteProviderInference( - { - sandboxName: "dcode-vllm-local", - model, - provider: "compatible-endpoint", - endpointUrl: "http://localhost:8000/v1", - credentialEnv: "COMPATIBLE_API_KEY", - }, - deps, - ), - ).resolves.toEqual({ done: false }); - - expect(deps.upsertProvider).toHaveBeenCalledWith( - "compatible-endpoint", - "openai", - "COMPATIBLE_API_KEY", - "http://host.openshell.internal:8000/v1", - { COMPATIBLE_API_KEY: "dummy-key" }, - ); - expect(deps.runOpenshell).toHaveBeenCalledWith( - [ - "inference", - "set", - "--no-verify", - "--provider", - "compatible-endpoint", - "--model", - model, - "--timeout", - "180", - ], - { ignoreError: true }, - ); - }); - - it.each([ - ["IPv6 loopback", "http://[::1]:8000/v1", "http://host.openshell.internal:8000/v1"], - ["default HTTP port", "http://localhost/v1", "http://host.openshell.internal/v1"], - [ - "first unprivileged port", - "http://localhost:1024/v1", - "http://host.openshell.internal:1024/v1", - ], - ])("rewrites %s compatible endpoint routes (#5744)", async (_name, endpointUrl, expectedUrl) => { - const deps = makeDeps(); - - await expect( - setupRemoteProviderInference( - { - sandboxName: "dcode-vllm-local", - model, - provider: "compatible-endpoint", - endpointUrl, - credentialEnv: "COMPATIBLE_API_KEY", - }, - deps, - ), - ).resolves.toEqual({ done: false }); - - expect(deps.upsertProvider).toHaveBeenCalledWith( - "compatible-endpoint", - "openai", - "COMPATIBLE_API_KEY", - expectedUrl, - { COMPATIBLE_API_KEY: "dummy-key" }, - ); - expect(deps.runOpenshell).toHaveBeenCalledWith( - [ - "inference", - "set", - "--no-verify", - "--provider", - "compatible-endpoint", - "--model", - model, - "--timeout", - "180", - ], - { ignoreError: true }, - ); - }); - - it("repairs an existing loopback compatible provider with the gateway alias (#5744)", async () => { - const deps = makeDeps(); - - await expect( - setupRemoteProviderInference( - { - sandboxName: "dcode-vllm-local", - model, - provider: "compatible-endpoint", - endpointUrl: "http://127.0.0.1:8000/v1", - credentialEnv: "COMPATIBLE_API_KEY", - reuseGatewayCredentialWithoutLocalKey: true, - }, - deps, - ), - ).resolves.toEqual({ done: false }); - - expect(deps.upsertProvider).toHaveBeenCalledWith( - "compatible-endpoint", - "openai", - "COMPATIBLE_API_KEY", - "http://host.openshell.internal:8000/v1", - {}, - ); - expect(deps.runOpenshell).toHaveBeenNthCalledWith( - 1, - ["provider", "get", "compatible-endpoint"], - { - ignoreError: true, - suppressOutput: true, - }, - ); - expect(deps.runOpenshell).toHaveBeenNthCalledWith( - 2, - [ - "inference", - "set", - "--no-verify", - "--provider", - "compatible-endpoint", - "--model", - model, - "--timeout", - "180", - ], - { ignoreError: true }, - ); - }); - - it("keeps the original provider route when reusing an existing non-loopback compatible provider (#5744)", async () => { - const deps = makeDeps(); - - await expect( - setupRemoteProviderInference( - { - sandboxName: "dcode-vllm-local", - model, - provider: "compatible-endpoint", - endpointUrl: "http://10.0.0.1:8000/v1", - credentialEnv: "COMPATIBLE_API_KEY", - reuseGatewayCredentialWithoutLocalKey: true, - }, - deps, - ), - ).resolves.toEqual({ done: false }); - - expect(deps.upsertProvider).not.toHaveBeenCalled(); - expect(deps.runOpenshell).toHaveBeenNthCalledWith( - 1, - ["provider", "get", "compatible-endpoint"], - { - ignoreError: true, - suppressOutput: true, - }, - ); - expect(deps.runOpenshell).toHaveBeenNthCalledWith( - 2, - [ - "inference", - "set", - "--no-verify", - "--provider", - "compatible-endpoint", - "--model", - model, - "--timeout", - "180", - ], - { ignoreError: true }, - ); - }); - - it.each([ - ["loopback route rewrite", "http://localhost:8000/v1", true], - ["non-loopback route", "http://10.0.0.1:8000/v1", false], - ])("uses --no-verify for %s only when the gateway URL differs (#5744)", async (_name, endpointUrl, expectedNoVerify) => { - const deps = makeDepsWithCompatibleSkipVerify(false); - - await expect( - setupRemoteProviderInference( - { - sandboxName: "dcode-vllm-local", - model, - provider: "compatible-endpoint", - endpointUrl, - credentialEnv: "COMPATIBLE_API_KEY", - }, - deps, - ), - ).resolves.toEqual({ done: false }); - - expect(deps.runOpenshell).toHaveBeenCalledWith( - [ - "inference", - "set", - ...(expectedNoVerify ? ["--no-verify"] : []), - "--provider", - "compatible-endpoint", - "--model", - model, - "--timeout", - "180", - ], - { ignoreError: true }, - ); - }); - - it.each([ - ["HTTPS loopback", "compatible-endpoint", "https://localhost:8000/v1"], - ["non-loopback host", "compatible-endpoint", "http://10.0.0.1:8000/v1"], - ["IPv4 privileged port", "compatible-endpoint", "http://localhost:1023/v1"], - ["IPv6 privileged port", "compatible-endpoint", "http://[::1]:1023/v1"], - ["IPv6 zone ID", "compatible-endpoint", "http://[::1%25eth0]:8000/v1"], - ["embedded URL credentials", "compatible-endpoint", "http://user:pass@localhost:8000/v1"], - ["malformed URL", "compatible-endpoint", "not a url"], - ["non-compatible provider", "openai-api", "http://localhost:8000/v1"], - ])("does not rewrite %s endpoints", async (_name, provider, endpointUrl) => { - const deps = makeDeps(); - const credentialEnv = - provider === "compatible-endpoint" ? "COMPATIBLE_API_KEY" : "OPENAI_API_KEY"; - - await expect( - setupRemoteProviderInference( - { - sandboxName: "dcode-vllm-local", - model, - provider, - endpointUrl, - credentialEnv, - }, - deps, - ), - ).resolves.toEqual({ done: false }); - - expect(deps.upsertProvider).toHaveBeenCalledWith( - provider, - "openai", - credentialEnv, - endpointUrl, - { [credentialEnv]: "dummy-key" }, - ); - expect(deps.runOpenshell).toHaveBeenCalledWith( - [ - "inference", - "set", - ...(provider === "compatible-endpoint" ? ["--no-verify"] : []), - "--provider", - provider, - "--model", - model, - ...(provider === "compatible-endpoint" ? ["--timeout", "180"] : []), - ], - { ignoreError: true }, - ); - }); -}); diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index d95cc5c53a..2cbc61331e 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -22,20 +22,9 @@ const { probeOpenAiLikeEndpoint } = require("../../inference/onboard-probes") as type StaleProviderReplaceResult = { ok: boolean; status?: number | null; message?: string }; -// #5744: compatible endpoints entered as host loopback URLs must be split -// across two routing contexts. The onboard smoke probe runs on the host, so it -// should keep the user-entered URL (`http://localhost:/v1`). The sandbox -// runtime reaches the same host service through OpenShell's sandbox host bridge, -// so the provider route uses `host.openshell.internal:` instead. -// -// Source boundary: OpenShell owns the gateway bridge and provider verification -// path. NemoClaw cannot infer every deployed gateway topology from provider -// config alone, so this helper applies only to the narrow, host-validated HTTP -// loopback shape that OpenShell exposes through the sandbox host bridge. -// -// Removal condition: drop this rewrite and the paired --no-verify once -// OpenShell verifies provider routes from the gateway/sandbox routing context, -// or exposes a definitive sandbox-path provider probe for this bridge. +// #5744: keep host-side validation on the user-entered loopback URL, but +// register the sandbox route through OpenShell's host bridge. Remove this when +// OpenShell can verify provider routes from the sandbox/gateway network context. function gatewayReachableCompatibleEndpointUrl( provider: string, endpointUrl: string | null | undefined, @@ -45,8 +34,6 @@ function gatewayReachableCompatibleEndpointUrl( try { parsed = new URL(endpointUrl); } catch { - // Leave malformed input untouched so the normal provider/probe validation - // path can report the URL problem instead of this classifier throwing first. return endpointUrl; } const hostname = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase(); @@ -367,11 +354,7 @@ export async function setupRemoteProviderInference( } const argsv = ["inference", "set"]; if (config.skipVerify || gatewayEndpointUrl !== resolvedEndpointUrl) { - // The host-side probe validates the user-provided loopback URL. When the - // provider route is rewritten to host.openshell.internal, OpenShell's - // host-side verifier cannot resolve that sandbox-only bridge name. - // See ensureLocalProviderReachable in local-inference-topology.ts for the - // local-provider counterpart and its removal condition. + // Host-side verification cannot resolve the sandbox-only bridge URL. argsv.push("--no-verify"); } argsv.push("--provider", provider, "--model", model);