diff --git a/docs/inference/local-compatible-inference-setup.mdx b/docs/inference/local-compatible-inference-setup.mdx index 1f0d7e8fdc..67e4543436 100644 --- a/docs/inference/local-compatible-inference-setup.mdx +++ b/docs/inference/local-compatible-inference-setup.mdx @@ -56,10 +56,12 @@ $$nemoclaw onboard When the wizard asks you to choose an inference provider, select **Other OpenAI-compatible endpoint**. Enter the base URL of your local server, for example `http://localhost:8000/v1` on the default Docker-driver topology. -For a containerized OpenShell gateway that cannot reach host `localhost`, use the host-gateway URL for your setup, commonly `http://host.openshell.internal:8000/v1`. +For HTTP loopback endpoints on the default HTTP port or an unprivileged port, such as `http://localhost:8000/v1`, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` so sandbox traffic can leave the container namespace. +This rewrite depends on an OpenShell topology that resolves `host.openshell.internal` inside the sandbox; if that bridge is unavailable, onboarding can still validate the host URL, but `$$nemoclaw status` is the authoritative runtime check. +Make sure the local server listens on an address reachable from containers, such as `0.0.0.0`; a server bound only to `127.0.0.1` can still be unreachable from the sandbox route. Port `8000` is included in NemoClaw's `local-inference` policy preset. -This is a sandbox-internal alias, so host-side endpoint probing is skipped during onboarding. -Use a routable endpoint when you need onboarding to verify the API, tool-calling, and streaming paths. +If you manually enter a sandbox-internal alias such as `http://host.openshell.internal:8000/v1`, host-side endpoint probing is skipped during onboarding. +Use a host-routable endpoint such as `localhost` when you need onboarding to verify the API, tool-calling, and streaming paths before gateway registration. Otherwise, confirm the runtime after onboarding with `$$nemoclaw status` and a short agent request. For GGUF models, start a compatible server such as `llama-server` yourself and enter the model id that server reports from `/v1/models`. @@ -73,7 +75,7 @@ Start llama.cpp on the host: ```bash llama-server \ -m /models/NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf \ - --host 127.0.0.1 \ + --host 0.0.0.0 \ --port 8000 \ -c 16384 \ -ngl 999 \ @@ -83,9 +85,10 @@ llama-server \ Select **Other OpenAI-compatible endpoint** during onboarding. Enter the server base URL, such as `http://localhost:8000/v1`, and the model id reported by `/v1/models`. -For a containerized OpenShell gateway, use the reachable host-service URL for your setup, commonly `http://host.openshell.internal:8000/v1`. -Host-side endpoint probing is skipped during onboarding for this sandbox-internal alias. -Use a routable endpoint when you need onboarding to verify the API, tool-calling, and streaming paths. +For HTTP loopback endpoints on the default HTTP port or an unprivileged port, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` for sandbox traffic. +This rewrite depends on an OpenShell topology that resolves `host.openshell.internal` inside the sandbox; if that bridge is unavailable, onboarding can still validate the host URL, but `$$nemoclaw status` is the authoritative runtime check. +If you manually enter a sandbox-internal alias such as `http://host.openshell.internal:8000/v1`, host-side endpoint probing is skipped during onboarding. +Use a host-routable endpoint such as `localhost` when you need onboarding to verify the API, tool-calling, and streaming paths. After onboarding, run `$$nemoclaw status` and check the `Inference` row before starting long agent work. diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index 0ff06a5ecc..2cbc61331e 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -22,6 +22,39 @@ const { probeOpenAiLikeEndpoint } = require("../../inference/onboard-probes") as type StaleProviderReplaceResult = { ok: boolean; status?: number | null; message?: string }; +// #5744: keep host-side validation on the user-entered loopback URL, but +// register the sandbox route through OpenShell's host bridge. Remove this when +// OpenShell can verify provider routes from the sandbox/gateway network context. +function gatewayReachableCompatibleEndpointUrl( + provider: string, + endpointUrl: string | null | undefined, +): string | null | undefined { + if (provider !== "compatible-endpoint" || !endpointUrl) return endpointUrl; + let parsed: URL; + try { + parsed = new URL(endpointUrl); + } catch { + return endpointUrl; + } + const hostname = parsed.hostname.replace(/^\[|\]$/g, "").toLowerCase(); + if (hostname.includes("%")) return endpointUrl; + const port = parsed.port ? Number(parsed.port) : null; + const isLoopback = hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1"; + if ( + parsed.protocol !== "http:" || + parsed.username || + parsed.password || + !isLoopback || + (port !== null && (!Number.isInteger(port) || port < 1024)) + ) { + return endpointUrl; + } + parsed.hostname = "host.openshell.internal"; + const pathname = parsed.pathname.replace(/\/+$/, ""); + parsed.pathname = pathname || "/"; + return parsed.pathname === "/" ? parsed.origin : `${parsed.origin}${parsed.pathname}`; +} + /** * Replace a provider that a prior Anthropic-Messages registration left behind * so it can be re-registered as `type=openai` for the OpenAI-compatible route @@ -206,6 +239,7 @@ export async function setupRemoteProviderInference( while (true) { const resolvedCredentialEnv = credentialEnv || (config && config.credentialEnv); const resolvedEndpointUrl = endpointUrl || (config && config.endpointUrl); + const gatewayEndpointUrl = gatewayReachableCompatibleEndpointUrl(provider, resolvedEndpointUrl); let providerResult; if (reuseGatewayCredentialWithoutLocalKey) { // This is only a last-moment existence probe. The primary authorization @@ -215,14 +249,23 @@ export async function setupRemoteProviderInference( ignoreError: true, suppressOutput: true, }); - providerResult = - existing.status === 0 - ? { ok: true } - : { - ok: false, - status: existing.status || 1, - message: `Recovered provider '${provider}' is no longer registered in OpenShell.`, - }; + if (existing.status !== 0) { + providerResult = { + ok: false, + status: existing.status || 1, + message: `Recovered provider '${provider}' is no longer registered in OpenShell.`, + }; + } else if (gatewayEndpointUrl !== resolvedEndpointUrl) { + providerResult = upsertProvider( + provider, + config.providerType, + resolvedCredentialEnv, + gatewayEndpointUrl, + {}, + ); + } else { + providerResult = { ok: true }; + } } else { const credentialValue = hydrateCredentialEnv(resolvedCredentialEnv); const env = @@ -285,7 +328,7 @@ export async function setupRemoteProviderInference( provider, config.providerType, resolvedCredentialEnv, - resolvedEndpointUrl, + gatewayEndpointUrl, env, ); } @@ -310,7 +353,8 @@ export async function setupRemoteProviderInference( return exitProcess(providerResult.status || 1); } const argsv = ["inference", "set"]; - if (config.skipVerify) { + if (config.skipVerify || gatewayEndpointUrl !== resolvedEndpointUrl) { + // Host-side verification cannot resolve the sandbox-only bridge URL. argsv.push("--no-verify"); } argsv.push("--provider", provider, "--model", model); diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index ab056e0016..73c46ff161 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -209,11 +209,17 @@ describe("inference setup navigation", () => { expect(section).toContain("[Use Ollama for Local Inference](use-local-inference)"); }); - it("uses a loopback-only bind for the raw model server example", () => { + it("uses a container-reachable bind for the raw model server example (#5744)", () => { const markdown = fs.readFileSync(selfHostedInferenceSetupPath, "utf8"); - expect(markdown).toContain("--host 127.0.0.1"); - expect(markdown).not.toContain("--host 0.0.0.0"); + expect(markdown).toContain("--host 0.0.0.0"); + expect(markdown).not.toContain("--host 127.0.0.1"); + expect(markdown).toContain( + "a server bound only to `127.0.0.1` can still be unreachable from the sandbox route.", + ); + expect(markdown).toContain( + "This rewrite depends on an OpenShell topology that resolves `host.openshell.internal` inside the sandbox", + ); }); it("routes vLLM tool-calling remediation to the self-hosted server guide", () => { @@ -279,10 +285,16 @@ describe("inference setup navigation", () => { expect(result.note).toContain("validation skipped"); expect(markdown).toContain("`http://host.openshell.internal:8000/v1`"); expect(markdown).toContain( - "This is a sandbox-internal alias, so host-side endpoint probing is skipped during onboarding.", + "For HTTP loopback endpoints on the default HTTP port or an unprivileged port, such as `http://localhost:8000/v1`, NemoClaw validates the endpoint from the host and registers the OpenShell gateway route through `host.openshell.internal:` so sandbox traffic can leave the container namespace.", + ); + expect(markdown).toContain( + "if that bridge is unavailable, onboarding can still validate the host URL, but `$$nemoclaw status` is the authoritative runtime check.", + ); + expect(markdown).toContain( + "If you manually enter a sandbox-internal alias such as `http://host.openshell.internal:8000/v1`, host-side endpoint probing is skipped during onboarding.", ); expect(markdown).toContain( - "Use a routable endpoint when you need onboarding to verify the API, tool-calling, and streaming paths.", + "Use a host-routable endpoint such as `localhost` when you need onboarding to verify the API, tool-calling, and streaming paths", ); const textAfterEachAlias = markdown.split(hostGatewayAlias).slice(1); expect(textAfterEachAlias).toHaveLength(2); diff --git a/test/onboard-inference-gateway-scope.test.ts b/test/onboard-inference-gateway-scope.test.ts index d45ddbff11..3762cd37fa 100644 --- a/test/onboard-inference-gateway-scope.test.ts +++ b/test/onboard-inference-gateway-scope.test.ts @@ -57,6 +57,55 @@ describe("onboarding inference gateway scope", () => { }); }); + it("registers loopback compatible endpoints through the gateway alias but keeps host smoke local (#5744)", async () => { + await withProcessEnv( + { COMPATIBLE_API_KEY: "sk-compatible-TEST-NOT-A-REAL-VALUE" }, + async () => { + const harness = createHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" ? { status: 1 } : undefined, + }); + const endpointUrl = "http://localhost:8000/v1"; + const model = "deepseek-ai/DeepSeek-V4-Flash"; + + await expect( + harness.setupInference( + "dcode-vllm-local", + model, + "compatible-endpoint", + endpointUrl, + "COMPATIBLE_API_KEY", + null, + [], + { gatewayName: GATEWAY }, + ), + ).resolves.toEqual({ ok: true }); + + expect(harness.commands.map(({ command }) => command)).toEqual([ + `provider get -g ${GATEWAY} compatible-endpoint`, + `provider create -g ${GATEWAY} --name compatible-endpoint --type openai --credential COMPATIBLE_API_KEY --config OPENAI_BASE_URL=http://host.openshell.internal:8000/v1`, + `inference set -g ${GATEWAY} --no-verify --provider compatible-endpoint --model ${model} --timeout 180`, + ]); + expect(harness.verifyOnboardInferenceSmoke).toHaveBeenCalledWith({ + provider: "compatible-endpoint", + model, + endpointUrl, + credentialEnv: "COMPATIBLE_API_KEY", + pinnedAddresses: [], + }); + expect(harness.updateSandbox).toHaveBeenCalledWith("dcode-vllm-local", { + provider: "compatible-endpoint", + model, + endpointUrl, + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: null, + gatewayName: GATEWAY, + }); + expectCommandsTargetOnly(harness.commands); + }, + ); + }); + it("keeps compatible-endpoint replacement and detach recovery on the target gateway", async () => { await withProcessEnv( { COMPATIBLE_ANTHROPIC_API_KEY: "sk-ant-TEST-NOT-A-REAL-VALUE" },