From b22a972ef59c42f46db1dbcf7573941006f4788b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 13:21:22 -0700 Subject: [PATCH 01/31] feat(bench): add progressive disclosure proof harness Signed-off-by: Aaron Erickson --- docs/index.yml | 6 + .../progressive-tool-disclosure-benchmark.mdx | 216 +++++ package.json | 1 + scripts/bench/README.md | 53 ++ scripts/bench/tool-disclosure/artifacts.ts | 112 +++ scripts/bench/tool-disclosure/assemble-run.ts | 144 +++ scripts/bench/tool-disclosure/catalog.ts | 734 +++++++++++++++ scripts/bench/tool-disclosure/drivers.ts | 166 ++++ scripts/bench/tool-disclosure/execute.ts | 624 +++++++++++++ scripts/bench/tool-disclosure/grading.ts | 72 ++ scripts/bench/tool-disclosure/mcp-server.ts | 286 ++++++ .../bench/tool-disclosure/openclaw-fixture.ts | 170 ++++ scripts/bench/tool-disclosure/quick-tunnel.ts | 142 +++ scripts/bench/tool-disclosure/recorder.ts | 838 ++++++++++++++++++ scripts/bench/tool-disclosure/report.ts | 252 ++++++ scripts/bench/tool-disclosure/run.ts | 714 +++++++++++++++ scripts/bench/tool-disclosure/schedule.ts | 192 ++++ scripts/bench/tool-disclosure/statistics.ts | 739 +++++++++++++++ scripts/bench/tool-disclosure/subprocess.ts | 105 +++ scripts/bench/tool-disclosure/tasks.ts | 368 ++++++++ scripts/bench/tool-disclosure/telemetry.ts | 165 ++++ scripts/bench/tool-disclosure/types.ts | 281 ++++++ scripts/bench/tool-disclosure/validation.ts | 358 ++++++++ test/bench/tool-disclosure-catalog.test.ts | 263 ++++++ test/bench/tool-disclosure-drivers.test.ts | 70 ++ test/bench/tool-disclosure-execute.test.ts | 116 +++ test/bench/tool-disclosure-fixture.test.ts | 194 ++++ test/bench/tool-disclosure-mcp-server.test.ts | 73 ++ .../tool-disclosure-quick-tunnel.test.ts | 32 + test/bench/tool-disclosure-recorder.test.ts | 457 ++++++++++ test/bench/tool-disclosure-report.test.ts | 413 +++++++++ test/bench/tool-disclosure-schedule.test.ts | 80 ++ test/bench/tool-disclosure-telemetry.test.ts | 61 ++ test/bench/tool-disclosure-validation.test.ts | 294 ++++++ 34 files changed, 8791 insertions(+) create mode 100644 docs/inference/progressive-tool-disclosure-benchmark.mdx create mode 100644 scripts/bench/tool-disclosure/artifacts.ts create mode 100644 scripts/bench/tool-disclosure/assemble-run.ts create mode 100644 scripts/bench/tool-disclosure/catalog.ts create mode 100644 scripts/bench/tool-disclosure/drivers.ts create mode 100644 scripts/bench/tool-disclosure/execute.ts create mode 100644 scripts/bench/tool-disclosure/grading.ts create mode 100644 scripts/bench/tool-disclosure/mcp-server.ts create mode 100644 scripts/bench/tool-disclosure/openclaw-fixture.ts create mode 100644 scripts/bench/tool-disclosure/quick-tunnel.ts create mode 100644 scripts/bench/tool-disclosure/recorder.ts create mode 100644 scripts/bench/tool-disclosure/report.ts create mode 100644 scripts/bench/tool-disclosure/run.ts create mode 100644 scripts/bench/tool-disclosure/schedule.ts create mode 100644 scripts/bench/tool-disclosure/statistics.ts create mode 100644 scripts/bench/tool-disclosure/subprocess.ts create mode 100644 scripts/bench/tool-disclosure/tasks.ts create mode 100644 scripts/bench/tool-disclosure/telemetry.ts create mode 100644 scripts/bench/tool-disclosure/types.ts create mode 100644 scripts/bench/tool-disclosure/validation.ts create mode 100644 test/bench/tool-disclosure-catalog.test.ts create mode 100644 test/bench/tool-disclosure-drivers.test.ts create mode 100644 test/bench/tool-disclosure-execute.test.ts create mode 100644 test/bench/tool-disclosure-fixture.test.ts create mode 100644 test/bench/tool-disclosure-mcp-server.test.ts create mode 100644 test/bench/tool-disclosure-quick-tunnel.test.ts create mode 100644 test/bench/tool-disclosure-recorder.test.ts create mode 100644 test/bench/tool-disclosure-report.test.ts create mode 100644 test/bench/tool-disclosure-schedule.test.ts create mode 100644 test/bench/tool-disclosure-telemetry.test.ts create mode 100644 test/bench/tool-disclosure-validation.test.ts diff --git a/docs/index.yml b/docs/index.yml index 8eea7f133f..cd4e206f0f 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -66,6 +66,9 @@ navigation: - page: "Model Capability Audit" path: _build/agent-variants/inference/model-capability-audit.openclaw.generated.mdx slug: model-capability-audit + - page: "Progressive Tool Disclosure Benchmark" + path: _build/agent-variants/inference/progressive-tool-disclosure-benchmark.openclaw.generated.mdx + slug: progressive-tool-disclosure-benchmark - page: "Switch Inference Providers" path: _build/agent-variants/inference/switch-inference-providers.openclaw.generated.mdx slug: switch-inference-providers @@ -249,6 +252,9 @@ navigation: - page: "Model Capability Audit" path: _build/agent-variants/inference/model-capability-audit.hermes.generated.mdx slug: model-capability-audit + - page: "Progressive Tool Disclosure Benchmark" + path: _build/agent-variants/inference/progressive-tool-disclosure-benchmark.hermes.generated.mdx + slug: progressive-tool-disclosure-benchmark - page: "Switch Inference Providers" path: _build/agent-variants/inference/switch-inference-providers.hermes.generated.mdx slug: switch-inference-providers diff --git a/docs/inference/progressive-tool-disclosure-benchmark.mdx b/docs/inference/progressive-tool-disclosure-benchmark.mdx new file mode 100644 index 0000000000..8a88b88aee --- /dev/null +++ b/docs/inference/progressive-tool-disclosure-benchmark.mdx @@ -0,0 +1,216 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Validate Progressive Tool Disclosure" +sidebar-title: "Progressive Tool Disclosure Benchmark" +description: "Run the reproducible two-campaign benchmark that validates progressive tool-disclosure claims across NemoClaw agent surfaces." +description-agent: "Explains how to prepare, run, and summarize the progressive tool-disclosure benchmark on a DGX Station. Use when validating tool-schema token, task-success, or latency claims for OpenClaw, Hermes, and LangChain Deep Agents Code." +keywords: "progressive tool disclosure benchmark, DGX Station benchmark, tool schema tokens, MCP benchmark" +position: 5 +--- + +This protocol compares direct tool exposure with progressive disclosure for OpenClaw, Hermes, and LangChain Deep Agents Code. +It produces checksummed evidence and only emits claim text that clears the configured statistical gates. + + +The repository does not include measured results. +Run both campaigns and retain their evidence before you make a performance or quality claim. + + +## Understand the frozen protocol + +The preparation command generates one deterministic catalog, task corpus, and paired execution schedule. +Use the generated files without adding, removing, or reordering runs. + +| Phase | Catalog tools | Tasks and repetitions per agent and mode | Purpose | +| --- | ---: | --- | --- | +| `static-visibility` | `16`, `64`, `256`, `512`, and `2209` | One exact capture at each size. | Measure the initial serialized tool-schema bytes, tokens, and visible tool count. | +| `small-control` | `64` | `24` tasks, one repetition. | Check behavior before context pressure becomes large. | +| `primary` | `512` | `24` tasks, five repetitions. | Supply the paired observations used by the public claim gates. | +| `large-stress` | `2209` | `8` tasks, one repetition. | Exercise the largest generated catalog without widening the primary claim. | + +The schedule contains `942` runs in each campaign and `1884` runs in total. +It keeps each progressive and direct pair adjacent while deterministically balancing which mode runs first. +The primary task set includes single-tool, ordered-chain, near-match, and no-tool cases. + +## Prepare the evidence directory + +Run preparation from a clean NemoClaw checkout at the exact system-under-test revision. +The sandbox base must use an immutable OCI digest; mutable tags such as `latest` are rejected. +Place the evidence directory outside the Git worktree so generated artifacts cannot change the recorded cleanliness state. + +```bash +npm run bench:tool-disclosure -- prepare \ + --output-dir \ + --sandbox-base ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:<64-hex-digest> +``` + +Preparation writes these inputs: + +- `catalog.json` contains the deterministic synthetic tool catalog. +- `primary-tasks.json` and `stress-tasks.json` contain the frozen task definitions. +- `schedule.json` contains both campaigns in execution order. +- `manifest.template.json` contains the public evidence schema fields that you must complete. +- The generated OpenClaw fixture directories contain native tool plugins for the scheduled catalog sizes. + +Copy `manifest.template.json` to `manifest.json` and replace every placeholder before the first run. + +```bash +cp /manifest.template.json /manifest.json +``` + +Record the full NemoClaw system-under-test and harness commit SHAs, exact model ID and revision, vLLM image digest and version, public serving flags, parser names, OpenShell version, agent versions, GPU and CPU description, driver and CUDA versions, RAM, and the observed power state. +Keep credentials, endpoint URLs, hostnames, usernames, reservation identifiers, and local paths out of the manifest. + + +The benchmark commands do not reserve Colossus infrastructure, start vLLM, or create sandboxes. +Prepare those resources first; the `execute` command starts the loopback recorder and synthetic MCP fixtures, drives the frozen schedule, and writes sanitized raw evidence plus derived run records. + + +## Reserve and preflight a DGX Station + +Use the current internal Colossus runbook to reserve a DGX Station and obtain host access. +Colossus pools, commands, and access routes are site-specific, so this public protocol does not prescribe an internal identifier or command spelling. + +The following lines are placeholders, not runnable commands: + +```text + + +``` + +On the reserved station, run the public preflight commands before starting inference. + +```bash +nvidia-smi +nvidia-smi -q +docker version +openshell --version +nemoclaw --version +git rev-parse HEAD +``` + +Confirm that the station has the expected GPU count and model, enough free GPU memory for the frozen model, the required container image by digest, and no competing GPU workload. +Set and record the intended power state before each campaign. +Do not publish raw Colossus output or raw preflight logs because they can contain internal host or reservation identifiers. + +## Route inference through the recorder + +Run one vLLM server and one benchmark recording proxy on the DGX Station. +Configure the NemoClaw managed inference route so every model request from every benchmark sandbox passes through the proxy before reaching vLLM. + +```text +OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes + | + NemoClaw managed inference route + | + recording proxy on 127.0.0.1, forwarding /v1/* + | + vLLM + /metrics and /tokenize +``` + +The recorder must bind to `127.0.0.1` and must not be exposed on a station or corporate network interface. +It forwards OpenAI-compatible `/v1` requests without following redirects and records content-free request metadata, including the canonical tool-schema byte count, schema hash, visible tool count, tool names, status, monotonic timing, and run sequence. +It does not retain prompts, messages, headers, request bodies, response bodies, model identifiers, endpoints, or errors from upstream bodies. + +Use vLLM `/tokenize` to count initial serialized tool-schema tokens with the exact served model tokenizer. +Read prompt and generation token counters from vLLM `/metrics` immediately before and after each scheduled run. +Keep benchmark concurrency at `1` so those process-level counter deltas remain attributable to one run. +Use temperature `0`, the manifest's frozen parser configuration, and the same public vLLM flags in both modes and both campaigns. + +## Run two fresh campaigns + +Treat the two campaigns as independent executions of the same prepared schedule. +Do not reuse an inference process, sandbox, model session, or recorder event stream across campaigns. + +Copy `execute-config.template.json` outside the evidence directory and fill in the campaign number, loopback vLLM and telemetry URLs, tokenizer model, fixed recorder port, live `vllm_container_name` and `vllm_container_id`, binary overrides when needed, plus `sandbox_names`, `sandbox_container_names`, and live `sandbox_instance_ids` mappings keyed as `::`. +The sandboxes' managed inference route must already target that recorder port. +Do not publish this file because it can contain local routing and sandbox identifiers. +Execution verifies the live vLLM container ID, Docker image ID, model and revision, parser names, public flags, and process start time. +It also verifies each live sandbox status against its expected agent, disclosure mode, immutable image ID, and instance ID. +Only hashes of private container identities and reviewed public configuration enter the evidence bundle. +This attestation path expects vLLM and the DGX Station's OpenShell sandboxes to use the Docker driver so `docker inspect` can verify the live state. + +```bash +npm run bench:tool-disclosure -- execute \ + --output-dir \ + --config +``` + +For campaign 1: + +1. Start a fresh vLLM process from the recorded image digest and flags. +2. Start a fresh loopback recorder and reset its event stream. +3. Create fresh OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes for the scheduled modes and fixtures. +4. Run `execute` for campaign 1. It appends each attempt atomically to `attempt-journal.jsonl`, records hashed live identities in `campaign-attestations.json`, and materializes content-free `raw-events.jsonl` plus one derived `nemoclaw.tool_disclosure_bench.v1` row per entry in `runs.jsonl`. +5. Stop vLLM, stop the recorder, and destroy the campaign sandboxes. + +For campaign 2, repeat the same five steps with another fresh vLLM process, recorder event stream, and sandbox set. +Use the same station class, system-under-test and harness revisions, model revision, image digest, parser configuration, and task artifacts. +If any of those controlled values changes, prepare a new benchmark ID and restart both campaigns. + +The agent adapters must use the generated deterministic handlers and verify expected tool names, order, argument hashes, call count, unnecessary calls, and the returned nonce. +Direct and progressive modes must differ only in their disclosure configuration. +Follow the retry allowance in `manifest.json` for setup failures, and do not convert task failures into setup failures. + +## Summarize and verify artifacts + +After all scheduled records are present, run the summarizer in the same evidence directory. + +```bash +npm run bench:tool-disclosure -- summarize --output-dir +``` + +The command consumes `manifest.json` and `runs.jsonl` and writes: + +- `summary.json` with paired aggregates, confidence intervals, and claim-gate decisions. +- `raw-events.jsonl` with content-free measurements from which every run record is re-derived. +- `attempt-journal.jsonl` as the crash-recoverable source for raw and derived run rows. +- `campaign-attestations.json` with vLLM process start times, hashed inference identities and configuration, and hashed live sandbox identities. +- `report.md` with environment metadata, measurements, gate status, and mechanically generated claim text. +- `evidence.json` with the public artifact inventory and SHA-256 values. +- `SHA256SUMS` with the checksum manifest. + +Verify the bundle on the DGX Station before you move it. + +```bash +cd +sha256sum -c SHA256SUMS +``` + +Treat every output as untrusted until you review it for credentials, endpoint URLs, hostnames, usernames, reservation identifiers, local paths, raw prompts, tool arguments, and response text. +Do not hand-edit `summary.json` or the validated claims in `report.md`. +If a correction changes an input or run record, regenerate the summary and checksums. + +## Apply the claim gates + +The summarizer uses a deterministic paired bootstrap with `10,000` samples. +It averages repetitions within each task first, then resamples whole progressive and direct task pairs, so the task is the statistical unit. + +The report can emit each claim only when its gate passes for OpenClaw, Hermes, and LangChain Deep Agents Code in both campaigns at the `512`-tool primary size. + +| Claim | Required gate in every campaign and agent cell | +| --- | --- | +| Initial serialized tool-schema tokens improved. | The paired progressive-minus-direct token difference has a `95%` confidence-interval upper bound below `0`, and the exact static capture has a positive token reduction. | +| Task success is noninferior. | The paired progressive-minus-direct success difference has a `95%` confidence-interval lower bound of at least `-5` percentage points. | +| End-to-end latency improved. | The paired progressive-minus-direct latency difference has a `95%` confidence-interval upper bound below `0`. | + +Repeated static captures must agree exactly on visible tool count, serialized bytes, and tokenizer tokens. +The summarizer excludes setup failures, but scores timeouts, incorrect tool behavior, model errors, and context overflows as task failures. +A blocked gate remains visible in `report.md`, and the report does not emit the corresponding cross-agent claim. + +## Interpret the scope + +The benchmark supports claims about the recorded commits, agents, model and serving configuration, DGX Station class, deterministic synthetic catalog, and task corpus. +It does not establish a universal result for other models, hardware, tool distributions, prompts, concurrency levels, or production workloads. + +Progressive disclosure is a context optimization, not an authorization boundary. +The OpenClaw fixture measures its native catalog path and does not establish that mcporter-managed MCP schemas are compacted. +The `2209`-tool stress phase supplies boundary evidence but does not widen claims beyond the `512`-tool primary phase. +Two fresh campaigns on one station class demonstrate repeatability, not cross-hardware generality. + +## Related topics + +- [Model Capability Audit](model-capability-audit) explains the evidence fields for model and agent compatibility claims. +- [Tool-Calling Reliability](tool-calling-reliability) separates tool-use behavior from endpoint connectivity failures. diff --git a/package.json b/package.json index 1665e6131e..1f38721bb5 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "test:projects:check": "tsx scripts/checks/vitest-project-overlap.ts", "test:titles:check": "tsx scripts/checks/test-title-style.ts", "bench": "tsx scripts/bench/run.ts", + "bench:tool-disclosure": "tsx scripts/bench/tool-disclosure/run.ts", "check": "npx prek run --all-files --stage pre-commit && npx prek run --all-files --stage manual", "check:diff": "npx prek run --from-ref origin/main --to-ref HEAD --stage pre-commit && npx commitlint --from origin/main --to HEAD && npx prek run --from-ref origin/main --to-ref HEAD --stage pre-push", "checks": "tsx scripts/checks/run.ts", diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 0977d9577d..0ae6766f9a 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -118,3 +118,56 @@ compared without exposing sandbox names or credentials. The harness exits non-zero when a selected metric errors, a supplied trace is invalid, or required prerequisites (endpoint, model, API key) are missing. + +## Progressive tool-disclosure benchmark + +The progressive tool-disclosure benchmark compares direct and progressive tool +visibility for OpenClaw, Hermes, and LangChain Deep Agents Code against one +deterministic synthetic catalog and task corpus. +It is a claim-validation harness, not an extension of the advisory value +benchmark above. + +Prepare a new evidence directory with the frozen catalog, task sets, schedule, +manifest template, and generated OpenClaw fixtures: + +```bash +npm run bench:tool-disclosure -- prepare \ + --output-dir \ + --sandbox-base ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:<64-hex-digest> +``` + +Run the emitted schedule as two fresh campaigns on the same recorded hardware +and inference configuration. +Each campaign requires a fresh vLLM process and fresh sandboxes. +Populate `manifest.json` from `manifest.template.json`, and append the public-safe +run records to `runs.jsonl`. + +Drive each prepared campaign with a private configuration that identifies the +fresh vLLM container and maps the frozen agent/mode/catalog cells to fresh +sandbox names, container names, and instance IDs: + +```bash +npm run bench:tool-disclosure -- execute \ + --output-dir --config +``` + +Execution uses a recoverable `attempt-journal.jsonl`, verifies the live vLLM +image/configuration and Docker-backed sandbox identities, writes only hashed +private identities to the attestation artifact, materializes content-free +`raw-events.jsonl`, and derives `runs.jsonl`; the summarizer rejects records that +cannot be reproduced from the journal and raw evidence. + +Summarize the completed evidence set: + +```bash +npm run bench:tool-disclosure -- summarize --output-dir +``` + +The summarizer writes `summary.json`, `report.md`, `evidence.json`, and +`SHA256SUMS`. +It derives public claim text mechanically and leaves claims blocked unless the +required confidence-interval gate passes for every agent in both campaigns. + +See the [progressive tool-disclosure benchmark protocol](../../docs/inference/progressive-tool-disclosure-benchmark.mdx) +for the DGX Station workflow, recorder topology, artifact rules, claim gates, +and limitations. diff --git a/scripts/bench/tool-disclosure/artifacts.ts b/scripts/bench/tool-disclosure/artifacts.ts new file mode 100644 index 0000000000..097e4ff276 --- /dev/null +++ b/scripts/bench/tool-disclosure/artifacts.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +const SAFE_ARTIFACT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u; + +export function ensureCampaignDirectory(outputDir: string, resume: boolean): string { + const resolved = path.resolve(outputDir); + if (fs.existsSync(resolved)) { + const stat = fs.lstatSync(resolved); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error("benchmark output path must be a real directory"); + } + const entries = fs.readdirSync(resolved); + if (!resume && entries.length > 0) { + throw new Error("benchmark output directory is not empty; pass --resume to reuse it"); + } + } else { + fs.mkdirSync(resolved, { recursive: true, mode: 0o700 }); + } + return resolved; +} + +export function artifactPath(outputDir: string, name: string): string { + if (!SAFE_ARTIFACT_NAME.test(name)) throw new Error(`invalid benchmark artifact name: ${name}`); + const root = path.resolve(outputDir); + const candidate = path.resolve(root, name); + if (path.dirname(candidate) !== root) + throw new Error("benchmark artifact escaped output directory"); + if (fs.existsSync(candidate) && fs.lstatSync(candidate).isSymbolicLink()) { + throw new Error(`benchmark artifact must not be a symlink: ${name}`); + } + return candidate; +} + +export function writeJsonArtifact(outputDir: string, name: string, value: unknown): string { + const destination = artifactPath(outputDir, name); + const temporary = `${destination}.${process.pid}.${Date.now()}.tmp`; + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + fs.renameSync(temporary, destination); + return destination; +} + +export function writeTextArtifact(outputDir: string, name: string, value: string): string { + const destination = artifactPath(outputDir, name); + const temporary = `${destination}.${process.pid}.${Date.now()}.tmp`; + fs.writeFileSync(temporary, value, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + fs.renameSync(temporary, destination); + return destination; +} + +export function appendJsonLine(outputDir: string, name: string, value: unknown): void { + const destination = artifactPath(outputDir, name); + fs.appendFileSync(destination, `${JSON.stringify(value)}\n`, { + encoding: "utf8", + mode: 0o600, + }); +} + +export function readJsonLines(filePath: string): T[] { + const raw = fs.readFileSync(filePath, "utf8"); + return raw + .split(/\r?\n/u) + .filter((line) => line.trim().length > 0) + .map((line, index) => { + try { + return JSON.parse(line) as T; + } catch { + throw new Error(`invalid JSONL at line ${index + 1}`); + } + }); +} + +export function sha256File(filePath: string): string { + return createHash("sha256").update(fs.readFileSync(filePath)).digest("hex"); +} + +export function writeChecksumManifest(outputDir: string, names: readonly string[]): string { + const unique = [...new Set(names)].sort(); + const lines = unique.map((name) => { + const file = artifactPath(outputDir, name); + if (!fs.statSync(file).isFile()) throw new Error(`cannot checksum non-file artifact: ${name}`); + return `${sha256File(file)} ${name}`; + }); + const destination = artifactPath(outputDir, "SHA256SUMS"); + fs.writeFileSync(destination, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); + return destination; +} + +export function scanArtifactsForForbiddenValues( + outputDir: string, + names: readonly string[], + forbiddenValues: readonly string[], +): void { + const forbidden = forbiddenValues.filter((value) => value.length > 0); + for (const name of names) { + const content = fs.readFileSync(artifactPath(outputDir, name), "utf8"); + const leaked = forbidden.find((value) => content.includes(value)); + if (leaked) throw new Error(`benchmark artifact ${name} contains a forbidden value`); + } +} diff --git a/scripts/bench/tool-disclosure/assemble-run.ts b/scripts/bench/tool-disclosure/assemble-run.ts new file mode 100644 index 0000000000..f3404a15a7 --- /dev/null +++ b/scripts/bench/tool-disclosure/assemble-run.ts @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { gradeTaskRun, type RecordedSyntheticCall } from "./grading"; +import type { ToolDisclosureRecordingEvent } from "./recorder"; +import type { ScheduledToolDisclosureRun } from "./schedule"; +import type { SyntheticBenchmarkTask } from "./tasks"; +import { + type BenchmarkRunOutcome, + type RunCorrectness, + TOOL_DISCLOSURE_SCHEMA_VERSION, + type ToolDisclosureManifest, + type ToolDisclosureRun, +} from "./types"; + +export interface RunInvocationResult { + exit_code: number | null; + timed_out: boolean; + elapsed_ms: number; + final_output: string; +} + +function successfulStaticCorrectness(): RunCorrectness { + return { + task_success: true, + expected_tool_names: true, + expected_tool_order: true, + expected_arguments: true, + expected_call_count: true, + nonce_present: true, + unnecessary_tool_calls: 0, + }; +} + +function failedCorrectness(callCount: number): RunCorrectness { + return { + task_success: false, + expected_tool_names: false, + expected_tool_order: false, + expected_arguments: false, + expected_call_count: false, + nonce_present: false, + unnecessary_tool_calls: callCount, + }; +} + +/** + * Convert transient model output plus content-free call/recorder events into a + * public-safe run record. Raw output and tool arguments never cross this API. + */ +export function assembleToolDisclosureRun(options: { + manifest: ToolDisclosureManifest; + scheduled: ScheduledToolDisclosureRun; + task?: SyntheticBenchmarkTask; + calls: readonly RecordedSyntheticCall[]; + recorderEvents: readonly ToolDisclosureRecordingEvent[]; + invocation: RunInvocationResult; + initialSchemaTokens: number; + promptTokens?: number; + completionTokens?: number; + failureOutcome?: Exclude; +}): ToolDisclosureRun { + if (!Number.isSafeInteger(options.initialSchemaTokens) || options.initialSchemaTokens < 0) { + throw new Error("initial schema token count must be a non-negative integer"); + } + const scheduledCampaign = `campaign-${options.scheduled.campaign}`; + if (!options.manifest.campaigns.some((item) => item.campaign_id === scheduledCampaign)) { + throw new Error(`scheduled campaign ${scheduledCampaign} is absent from the manifest`); + } + if (options.scheduled.phase !== "static-visibility" && !options.task) { + throw new Error("task runs require their frozen task definition"); + } + if (options.task && options.scheduled.task_id !== options.task.id) { + throw new Error("scheduled task does not match the supplied task definition"); + } + + const modelEvents = options.recorderEvents.filter((event) => event.model_call_sequence !== null); + const initial = modelEvents.find((event) => event.model_call_sequence === 1); + let outcome: BenchmarkRunOutcome; + let correctness: RunCorrectness; + if (options.failureOutcome) { + outcome = options.failureOutcome; + correctness = failedCorrectness(options.calls.length); + } else if ( + options.scheduled.phase === "static-visibility" && + initial && + options.initialSchemaTokens > 0 + ) { + outcome = "success"; + correctness = successfulStaticCorrectness(); + } else if (options.invocation.timed_out) { + outcome = "timeout"; + correctness = failedCorrectness(options.calls.length); + } else if (options.invocation.exit_code !== 0 || !initial || initial.outcome !== "completed") { + outcome = "model-error"; + correctness = failedCorrectness(options.calls.length); + } else { + const graded = gradeTaskRun( + options.task as SyntheticBenchmarkTask, + options.calls, + options.invocation.final_output, + ); + outcome = graded.outcome; + correctness = graded.correctness; + } + + return { + schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, + benchmark_id: options.manifest.benchmark_id, + campaign_id: scheduledCampaign, + run_id: options.scheduled.run_id, + phase: options.scheduled.phase, + agent: options.scheduled.agent, + mode: options.scheduled.mode, + catalog_size: options.scheduled.catalog_size, + task_id: options.task?.id ?? "static-capture", + ...(options.task ? { task_kind: options.task.kind } : {}), + repetition: options.scheduled.repetition, + execution_seed: options.manifest.protocol.execution_seed, + outcome, + scored: options.scheduled.phase !== "static-visibility" && outcome !== "setup-error", + correctness, + measurements: { + initial_tool_schema: { + tool_count: initial?.visible_tool_count ?? 0, + serialized_bytes: initial?.canonical_tools_json_bytes ?? 0, + tokenizer_tokens: options.initialSchemaTokens, + }, + ...(options.promptTokens === undefined ? {} : { total_prompt_tokens: options.promptTokens }), + ...(options.completionTokens === undefined + ? {} + : { completion_tokens: options.completionTokens }), + ...(initial?.streaming !== true || + initial.time_to_first_byte_ms === null || + initial.time_to_first_byte_ms === undefined + ? {} + : { time_to_first_response_byte_ms: initial.time_to_first_byte_ms }), + inference_time_ms: modelEvents.reduce((total, event) => total + event.duration_ms, 0), + end_to_end_time_ms: options.invocation.elapsed_ms, + model_calls: modelEvents.length, + discovery_calls: Math.max(0, modelEvents.length - options.calls.length - 1), + }, + }; +} diff --git a/scripts/bench/tool-disclosure/catalog.ts b/scripts/bench/tool-disclosure/catalog.ts new file mode 100644 index 0000000000..3c7f024457 --- /dev/null +++ b/scripts/bench/tool-disclosure/catalog.ts @@ -0,0 +1,734 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +export const TOOL_DISCLOSURE_CATALOG_SCHEMA_VERSION = + "nemoclaw.tool_disclosure.catalog.v1" as const; +export const DEFAULT_SYNTHETIC_CATALOG_SEED = "nemoclaw-tool-disclosure-v1"; +export const MAX_SYNTHETIC_TOOLS = 2_209; + +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; +export interface JsonObject { + [key: string]: JsonValue; +} + +export interface JsonSchema { + type: "object" | "array" | "string" | "integer" | "number" | "boolean"; + description?: string; + properties?: Readonly>; + required?: readonly string[]; + additionalProperties?: boolean; + items?: JsonSchema; + enum?: readonly JsonPrimitive[]; + minLength?: number; + maxLength?: number; + pattern?: string; + minimum?: number; + maximum?: number; + minItems?: number; + maxItems?: number; + uniqueItems?: boolean; +} + +export interface OpenAIFunctionTool { + type: "function"; + function: { + name: string; + description: string; + parameters: JsonSchema; + }; +} + +/** + * The categories are deliberately synthetic and vendor-neutral. Their labels, + * resources, and operations make every generated tool understandable without + * implying access to a real service or copying a production API. + */ +export const SYNTHETIC_CATEGORIES = [ + { + id: "calendar", + label: "calendar", + resource: "event", + purpose: "appointments, availability, and event logistics", + operations: [ + "find", + "schedule", + "reschedule", + "cancel", + "list_attendees", + "check_availability", + ], + }, + { + id: "contacts", + label: "contacts", + resource: "contact", + purpose: "people and organization directory records", + operations: ["find", "create", "update", "merge", "list_groups", "verify"], + }, + { + id: "email", + label: "email", + resource: "message", + purpose: "mailbox messages and threads", + operations: ["search", "draft", "send", "archive", "summarize", "list_attachments"], + }, + { + id: "documents", + label: "documents", + resource: "document", + purpose: "text documents and their revisions", + operations: ["search", "create", "update", "compare", "export", "list_revisions"], + }, + { + id: "storage", + label: "storage", + resource: "file", + purpose: "files, folders, and object metadata", + operations: ["find", "upload", "copy", "move", "share", "inspect_metadata"], + }, + { + id: "messaging", + label: "messaging", + resource: "conversation", + purpose: "chat conversations and channel messages", + operations: ["search", "post", "reply", "list_members", "summarize", "pin"], + }, + { + id: "projects", + label: "projects", + resource: "project", + purpose: "project plans, milestones, and ownership", + operations: ["find", "create", "update", "list_milestones", "assign_owner", "summarize"], + }, + { + id: "issues", + label: "issues", + resource: "issue", + purpose: "work items, defects, and status tracking", + operations: ["search", "create", "update", "link", "list_blockers", "triage"], + }, + { + id: "source_control", + label: "source control", + resource: "revision", + purpose: "repositories, revisions, and code review metadata", + operations: ["search", "inspect", "compare", "list_changes", "list_reviews", "summarize"], + }, + { + id: "ci_cd", + label: "continuous integration", + resource: "pipeline", + purpose: "build pipelines, jobs, and deployment runs", + operations: ["find", "inspect", "retry", "cancel", "list_jobs", "summarize"], + }, + { + id: "observability", + label: "observability", + resource: "signal", + purpose: "metrics, logs, traces, and service health", + operations: ["query", "inspect", "compare", "list_alerts", "summarize", "correlate"], + }, + { + id: "incidents", + label: "incidents", + resource: "incident", + purpose: "operational incidents and response coordination", + operations: ["find", "create", "update", "list_responders", "summarize", "close"], + }, + { + id: "infrastructure", + label: "infrastructure", + resource: "resource", + purpose: "deployed infrastructure and configuration state", + operations: ["find", "inspect", "compare", "list_dependencies", "validate", "summarize"], + }, + { + id: "compute", + label: "compute", + resource: "compute allocation", + purpose: "compute capacity, allocations, and runtime state", + operations: ["find", "inspect", "list_capacity", "compare", "validate", "summarize"], + }, + { + id: "databases", + label: "databases", + resource: "dataset", + purpose: "database objects, queries, and metadata", + operations: ["find", "describe", "query", "list_columns", "validate", "summarize"], + }, + { + id: "analytics", + label: "analytics", + resource: "analysis", + purpose: "reports, measures, and analytical results", + operations: ["find", "calculate", "compare", "list_dimensions", "forecast", "summarize"], + }, + { + id: "finance", + label: "finance", + resource: "financial record", + purpose: "budgets, transactions, and financial summaries", + operations: ["find", "calculate", "compare", "list_transactions", "validate", "summarize"], + }, + { + id: "commerce", + label: "commerce", + resource: "order", + purpose: "orders, products, and fulfillment records", + operations: ["find", "create", "update", "list_items", "track", "summarize"], + }, + { + id: "travel", + label: "travel", + resource: "itinerary", + purpose: "bookings, itineraries, and travel options", + operations: ["search", "inspect", "compare", "list_segments", "validate", "summarize"], + }, + { + id: "maps", + label: "maps", + resource: "place", + purpose: "places, routes, and geographic information", + operations: ["search", "inspect", "route", "list_nearby", "compare", "summarize"], + }, + { + id: "weather", + label: "weather", + resource: "forecast", + purpose: "forecasts, conditions, and weather alerts", + operations: ["find", "inspect", "compare", "list_alerts", "validate", "summarize"], + }, + { + id: "research", + label: "research", + resource: "source", + purpose: "research sources, evidence, and citations", + operations: ["search", "inspect", "compare", "list_citations", "validate", "summarize"], + }, + { + id: "knowledge", + label: "knowledge", + resource: "knowledge entry", + purpose: "knowledge-base entries and linked concepts", + operations: ["search", "inspect", "create", "update", "list_links", "summarize"], + }, + { + id: "media", + label: "media", + resource: "media asset", + purpose: "audio, image, and video asset metadata", + operations: ["search", "inspect", "transcode", "list_variants", "compare", "summarize"], + }, +] as const; + +export type SyntheticCategoryId = (typeof SYNTHETIC_CATEGORIES)[number]["id"]; +export type SchemaComplexity = "small" | "medium" | "large"; + +export interface SyntheticTool { + index: number; + category: SyntheticCategoryId; + operation: string; + complexity: SchemaComplexity; + definition: OpenAIFunctionTool; + handler: { + kind: "deterministic_fixture_v1"; + key: string; + }; +} + +export interface SyntheticCatalog { + schema_version: typeof TOOL_DISCLOSURE_CATALOG_SCHEMA_VERSION; + seed: string; + size: number; + max_size: typeof MAX_SYNTHETIC_TOOLS; + tools_sha256: string; + tools: SyntheticTool[]; +} + +export interface SyntheticCatalogOptions { + size?: number; + seed?: string | number; +} + +export interface SyntheticToolResult extends JsonObject { + ok: true; + tool_name: string; + category: SyntheticCategoryId; + operation: string; + nonce: string; + arguments_sha256: string; + summary: string; +} + +export class SyntheticToolInvocationError extends Error {} + +const TOOL_NAME_PATTERN = /^[A-Za-z0-9_-]{1,64}$/; +// A 25/50/25 cycle makes the medium schema the catalog median while retaining +// substantial small and large populations for scaling comparisons. The medium +// definition is intentionally near the benchmark's roughly 400-token target; +// authoritative token counts still come from the pinned model tokenizer. +const COMPLEXITY_CYCLE: readonly SchemaComplexity[] = [ + "small", + "small", + "small", + "small", + "small", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "large", + "large", + "large", + "large", + "large", +]; + +export function canonicalJson(value: JsonValue): string { + return canonicalize(value, new Set()); +} + +function canonicalize(value: JsonValue, ancestors: Set): string { + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new TypeError("canonical JSON does not support non-finite numbers"); + return JSON.stringify(value); + } + if (typeof value !== "object") { + throw new TypeError(`canonical JSON does not support ${typeof value}`); + } + if (ancestors.has(value)) throw new TypeError("canonical JSON does not support cyclic values"); + + ancestors.add(value); + try { + if (Array.isArray(value)) { + return `[${value.map((item) => canonicalize(item, ancestors)).join(",")}]`; + } + const members = Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalize(value[key], ancestors)}`); + return `{${members.join(",")}}`; + } finally { + ancestors.delete(value); + } +} + +export function sha256Hex(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +function normalizeSeed(seed: string | number): string { + if (typeof seed === "number" && (!Number.isSafeInteger(seed) || seed < 0)) { + throw new TypeError("catalog seed numbers must be non-negative safe integers"); + } + const normalized = String(seed); + if (normalized.length === 0) throw new TypeError("catalog seed must not be empty"); + return normalized; +} + +function hashUint32(seed: string, scope: string): number { + const digest = createHash("sha256").update(`${seed}\0${scope}`, "utf8").digest(); + return digest.readUInt32BE(0); +} + +function shuffledCategories(seed: string): readonly (typeof SYNTHETIC_CATEGORIES)[number][] { + const categories = [...SYNTHETIC_CATEGORIES]; + for (let index = categories.length - 1; index > 0; index -= 1) { + const swap = hashUint32(seed, `category-permutation:${index}`) % (index + 1); + [categories[index], categories[swap]] = [categories[swap], categories[index]]; + } + return categories; +} + +function complexityForIndex(index: number): SchemaComplexity { + return COMPLEXITY_CYCLE[index % COMPLEXITY_CYCLE.length]; +} + +function resourceIdSchema(resource: string): JsonSchema { + return { + type: "string", + description: `Stable synthetic ${resource} identifier. A prior tool nonce is also valid.`, + minLength: 1, + maxLength: 96, + pattern: "^[A-Za-z0-9][A-Za-z0-9_.:-]*$", + }; +} + +function buildParameters( + category: (typeof SYNTHETIC_CATEGORIES)[number], + operation: string, + complexity: SchemaComplexity, +): JsonSchema { + const properties: Record = { + resource_id: resourceIdSchema(category.resource), + query: { + type: "string", + description: `Precise synthetic request for the ${operation} operation.`, + minLength: 1, + maxLength: 160, + }, + }; + const required = ["resource_id", "query"]; + + if (complexity !== "small") { + properties.limit = { + type: "integer", + description: "Maximum number of deterministic fixture records to consider.", + minimum: 1, + maximum: 50, + }; + properties.include_archived = { + type: "boolean", + description: "Whether archived synthetic records participate in the operation.", + }; + properties.tags = { + type: "array", + description: "Optional synthetic labels used to narrow the operation.", + items: { type: "string", minLength: 1, maxLength: 32 }, + maxItems: 5, + uniqueItems: true, + }; + required.push("limit"); + } + + if (complexity === "large") { + properties.filters = { + type: "object", + description: `Structured filters for the synthetic ${category.resource}.`, + properties: { + status: { + type: "string", + enum: ["active", "pending", "complete"], + description: "Synthetic lifecycle state to select.", + }, + owner: { + type: "string", + minLength: 1, + maxLength: 48, + description: "Synthetic owner identifier.", + }, + window: { + type: "object", + description: "Inclusive logical-day range within the fixture timeline.", + properties: { + start_day: { type: "integer", minimum: 1, maximum: 365 }, + end_day: { type: "integer", minimum: 1, maximum: 365 }, + }, + required: ["start_day", "end_day"], + additionalProperties: false, + }, + }, + required: ["status", "owner", "window"], + additionalProperties: false, + }; + properties.options = { + type: "object", + description: "Deterministic projection and ordering controls.", + properties: { + sort_order: { type: "string", enum: ["ascending", "descending"] }, + fields: { + type: "array", + items: { type: "string", minLength: 1, maxLength: 32 }, + minItems: 1, + maxItems: 6, + uniqueItems: true, + }, + explain: { type: "boolean" }, + }, + required: ["sort_order", "fields", "explain"], + additionalProperties: false, + }; + properties.correlation_id = { + type: "string", + description: "Caller-provided identifier for deterministic chain assertions.", + minLength: 1, + maxLength: 64, + pattern: "^[A-Za-z0-9_.:-]+$", + }; + required.push("filters", "options"); + } + + return { + type: "object", + description: `Inputs for a synthetic ${category.label} benchmark operation.`, + properties, + required, + additionalProperties: false, + }; +} + +function generateTool( + index: number, + seed: string, + categories: readonly (typeof SYNTHETIC_CATEGORIES)[number][], +): SyntheticTool { + const category = categories[index % categories.length]; + const occurrence = Math.floor(index / categories.length); + const operationOffset = + hashUint32(seed, `operation-offset:${category.id}`) % category.operations.length; + const operation = + category.operations[(occurrence + operationOffset) % category.operations.length]; + const complexity = complexityForIndex(index); + const name = `bench_${category.id}_${operation}_${index.toString().padStart(4, "0")}`; + + if (!TOOL_NAME_PATTERN.test(name)) throw new Error(`generated unsafe tool name: ${name}`); + + return { + index, + category: category.id, + operation, + complexity, + definition: { + type: "function", + function: { + name, + description: [ + `Synthetic ${category.label} fixture: ${operation.replaceAll("_", " ")} one ${category.resource}.`, + `Use for benchmark requests about ${category.purpose}; it never contacts a real service.`, + ].join(" "), + parameters: buildParameters(category, operation, complexity), + }, + }, + handler: { + kind: "deterministic_fixture_v1", + key: sha256Hex(`${seed}\0handler\0${index}`).slice(0, 24), + }, + }; +} + +function assertCatalogSize(size: number): void { + if (!Number.isInteger(size) || size < 1 || size > MAX_SYNTHETIC_TOOLS) { + throw new RangeError(`catalog size must be an integer from 1 through ${MAX_SYNTHETIC_TOOLS}`); + } +} + +export function generateSyntheticTools( + size = MAX_SYNTHETIC_TOOLS, + seed: string | number = DEFAULT_SYNTHETIC_CATALOG_SEED, +): SyntheticTool[] { + assertCatalogSize(size); + const normalizedSeed = normalizeSeed(seed); + const categories = shuffledCategories(normalizedSeed); + return Array.from({ length: size }, (_, index) => + generateTool(index, normalizedSeed, categories), + ); +} + +export function generateSyntheticCatalog(options: SyntheticCatalogOptions = {}): SyntheticCatalog { + const size = options.size ?? MAX_SYNTHETIC_TOOLS; + const seed = normalizeSeed(options.seed ?? DEFAULT_SYNTHETIC_CATALOG_SEED); + const tools = generateSyntheticTools(size, seed); + return { + schema_version: TOOL_DISCLOSURE_CATALOG_SCHEMA_VERSION, + seed, + size, + max_size: MAX_SYNTHETIC_TOOLS, + tools_sha256: sha256Hex(canonicalJson(tools as unknown as JsonValue)), + tools, + }; +} + +export function generateCatalogPrefix(catalog: SyntheticCatalog, size: number): SyntheticCatalog { + assertCatalogSize(size); + if (size > catalog.size) { + throw new RangeError(`catalog prefix ${size} exceeds source catalog size ${catalog.size}`); + } + const tools = catalog.tools.slice(0, size); + return { + ...catalog, + size, + tools_sha256: sha256Hex(canonicalJson(tools as unknown as JsonValue)), + tools, + }; +} + +export function generateCatalogPrefixes( + catalog: SyntheticCatalog, + sizes: readonly number[], +): SyntheticCatalog[] { + return sizes.map((size) => generateCatalogPrefix(catalog, size)); +} + +export function toOpenAIFunctionTools(catalog: SyntheticCatalog): OpenAIFunctionTool[] { + return catalog.tools.map((tool) => tool.definition); +} + +export function buildSyntheticArguments( + tool: SyntheticTool, + variant = 0, + overrides: JsonObject = {}, +): JsonObject { + if (!Number.isSafeInteger(variant) || variant < 0) { + throw new TypeError("argument variant must be a non-negative safe integer"); + } + const argumentsObject: JsonObject = { + resource_id: `${tool.category}-${tool.index}-${variant}`, + query: `${tool.operation.replaceAll("_", " ")} fixture request ${variant}`, + }; + if (tool.complexity !== "small") { + argumentsObject.limit = 3 + (variant % 5); + argumentsObject.include_archived = variant % 2 === 0; + argumentsObject.tags = [`fixture-${variant % 7}`, tool.category]; + } + if (tool.complexity === "large") { + const startDay = 1 + (variant % 300); + argumentsObject.filters = { + status: (["active", "pending", "complete"] as const)[variant % 3], + owner: `owner-${variant % 11}`, + window: { start_day: startDay, end_day: startDay + 7 }, + }; + argumentsObject.options = { + sort_order: variant % 2 === 0 ? "ascending" : "descending", + fields: ["id", "status", "updated_at"], + explain: true, + }; + argumentsObject.correlation_id = `correlation-${tool.index}-${variant}`; + } + return { ...argumentsObject, ...overrides }; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function validateAgainstSchema(value: unknown, schema: JsonSchema, path: string): string[] { + if (schema.type === "object") { + if (!isPlainObject(value)) return [`${path} must be an object`]; + const properties = schema.properties ?? {}; + const errors: string[] = []; + for (const required of schema.required ?? []) { + if (!(required in value)) errors.push(`${path}.${required} is required`); + } + if (schema.additionalProperties === false) { + for (const key of Object.keys(value)) { + if (!(key in properties)) errors.push(`${path}.${key} is not allowed`); + } + } + for (const [key, child] of Object.entries(properties)) { + if (key in value) errors.push(...validateAgainstSchema(value[key], child, `${path}.${key}`)); + } + return errors; + } + if (schema.type === "array") { + if (!Array.isArray(value)) return [`${path} must be an array`]; + const errors: string[] = []; + if (schema.minItems !== undefined && value.length < schema.minItems) { + errors.push(`${path} must contain at least ${schema.minItems} item(s)`); + } + if (schema.maxItems !== undefined && value.length > schema.maxItems) { + errors.push(`${path} must contain at most ${schema.maxItems} item(s)`); + } + if ( + schema.uniqueItems && + new Set(value.map((item) => canonicalJson(item as JsonValue))).size !== value.length + ) { + errors.push(`${path} items must be unique`); + } + if (schema.items) { + value.forEach((item, index) => { + errors.push( + ...validateAgainstSchema(item, schema.items as JsonSchema, `${path}[${index}]`), + ); + }); + } + return errors; + } + if (schema.type === "string") { + if (typeof value !== "string") return [`${path} must be a string`]; + const errors: string[] = []; + if (schema.minLength !== undefined && value.length < schema.minLength) + errors.push(`${path} is too short`); + if (schema.maxLength !== undefined && value.length > schema.maxLength) + errors.push(`${path} is too long`); + if (schema.pattern && !new RegExp(schema.pattern).test(value)) + errors.push(`${path} has an invalid format`); + if (schema.enum && !schema.enum.includes(value)) + errors.push(`${path} must be an allowed value`); + return errors; + } + if (schema.type === "boolean") + return typeof value === "boolean" ? [] : [`${path} must be a boolean`]; + if (schema.type === "integer" || schema.type === "number") { + if (typeof value !== "number" || !Number.isFinite(value)) return [`${path} must be a number`]; + const errors: string[] = []; + if (schema.type === "integer" && !Number.isInteger(value)) + errors.push(`${path} must be an integer`); + if (schema.minimum !== undefined && value < schema.minimum) + errors.push(`${path} is below its minimum`); + if (schema.maximum !== undefined && value > schema.maximum) + errors.push(`${path} is above its maximum`); + return errors; + } + return [`${path} uses an unsupported schema type`]; +} + +export function validateSyntheticArguments(tool: SyntheticTool, args: unknown): string[] { + return validateAgainstSchema(args, tool.definition.function.parameters, "arguments"); +} + +export function syntheticResultNonce(tool: SyntheticTool, args: JsonObject): string { + const errors = validateSyntheticArguments(tool, args); + if (errors.length > 0) throw new SyntheticToolInvocationError(errors.join("; ")); + return `nonce_${sha256Hex(`${tool.handler.key}\0${canonicalJson(args)}`).slice(0, 20)}`; +} + +export function executeSyntheticTool(tool: SyntheticTool, args: JsonObject): SyntheticToolResult { + const argumentsSha256 = sha256Hex(canonicalJson(args)); + return { + ok: true, + tool_name: tool.definition.function.name, + category: tool.category, + operation: tool.operation, + nonce: syntheticResultNonce(tool, args), + arguments_sha256: argumentsSha256, + summary: `${tool.operation.replaceAll("_", " ")} completed for ${String(args.resource_id)}`, + }; +} + +export function validateSyntheticCatalog(catalog: SyntheticCatalog): string[] { + const errors: string[] = []; + if (catalog.schema_version !== TOOL_DISCLOSURE_CATALOG_SCHEMA_VERSION) { + errors.push(`unsupported catalog schema version: ${catalog.schema_version}`); + } + if (catalog.size !== catalog.tools.length) errors.push("catalog size does not match tool count"); + if (catalog.max_size !== MAX_SYNTHETIC_TOOLS) errors.push("catalog max_size is not canonical"); + + const names = new Set(); + catalog.tools.forEach((tool, index) => { + const name = tool.definition.function.name; + if (tool.index !== index) + errors.push(`tool at position ${index} has non-contiguous index ${tool.index}`); + if (!TOOL_NAME_PATTERN.test(name)) errors.push(`tool ${index} has unsafe name ${name}`); + if (names.has(name)) errors.push(`duplicate tool name: ${name}`); + names.add(name); + if (tool.definition.type !== "function") errors.push(`tool ${index} is not a function tool`); + if (tool.definition.function.parameters.additionalProperties !== false) { + errors.push(`tool ${index} parameters must reject additional properties`); + } + if (tool.handler.kind !== "deterministic_fixture_v1") { + errors.push(`tool ${index} has unsupported handler kind`); + } + }); + + const expectedHash = sha256Hex(canonicalJson(catalog.tools as unknown as JsonValue)); + if (catalog.tools_sha256 !== expectedHash) + errors.push("catalog tools_sha256 does not match tools"); + return errors; +} + +export function assertValidSyntheticCatalog(catalog: SyntheticCatalog): void { + const errors = validateSyntheticCatalog(catalog); + if (errors.length > 0) throw new Error(`invalid synthetic catalog: ${errors.join("; ")}`); +} diff --git a/scripts/bench/tool-disclosure/drivers.ts b/scripts/bench/tool-disclosure/drivers.ts new file mode 100644 index 0000000000..6c9d59655b --- /dev/null +++ b/scripts/bench/tool-disclosure/drivers.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; + +import type { ToolDisclosureAgent } from "./schedule"; + +export interface AgentDriverCommand { + command: string; + args: string[]; + redactions: string[]; +} + +export const OPENCLAW_BENCH_CALLS_PATH = "/sandbox/.nemoclaw-benchmark/calls.jsonl"; + +function validateIdentifier(value: string, label: string): void { + if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/u.test(value)) { + throw new Error(`${label} contains unsupported characters`); + } +} + +function base64(value: string): string { + return Buffer.from(value, "utf8").toString("base64"); +} + +function shellScriptFor(agent: ToolDisclosureAgent, prompt: string, sessionId: string): string { + const encodedPrompt = base64(prompt); + if (agent === "openclaw") { + return [ + "set -eu", + `prompt=$(printf '%s' '${encodedPrompt}' | base64 -d)`, + `exec openclaw agent --agent main --json --thinking off --session-id '${sessionId}' -m "$prompt"`, + ].join("; "); + } + if (agent === "langchain-deepagents-code") { + return [ + "set -eu", + `prompt=$(printf '%s' '${encodedPrompt}' | base64 -d)`, + 'exec nemoclaw-start dcode -n "$prompt"', + ].join("; "); + } + const payload = base64( + JSON.stringify({ + messages: [{ role: "user", content: prompt }], + max_tokens: 512, + temperature: 0, + stream: false, + }), + ); + return [ + "set -eu", + "set -a", + "[ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env", + "set +a", + `payload=$(printf '%s' '${payload}' | base64 -d)`, + 'if [ -n "${API_SERVER_KEY:-}" ]; then', + ' exec curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H \'Content-Type: application/json\' -H "Authorization: Bearer ${API_SERVER_KEY}" --data-binary "$payload"', + "fi", + "exec curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary \"$payload\"", + ].join("; "); +} + +export function buildAgentDriverCommand(options: { + openshellBin?: string; + sandboxName: string; + agent: ToolDisclosureAgent; + prompt: string; + sessionId: string; +}): AgentDriverCommand { + validateIdentifier(options.sandboxName, "sandboxName"); + validateIdentifier(options.sessionId, "sessionId"); + if (!options.prompt.trim()) throw new Error("benchmark task prompt must not be empty"); + const script = shellScriptFor(options.agent, options.prompt, options.sessionId); + return { + command: options.openshellBin ?? "openshell", + args: ["sandbox", "exec", "-n", options.sandboxName, "--", "sh", "-lc", script], + redactions: [ + options.prompt, + base64(options.prompt), + base64( + JSON.stringify({ + messages: [{ role: "user", content: options.prompt }], + max_tokens: 512, + temperature: 0, + stream: false, + }), + ), + ], + }; +} + +export function outputContainsOracle(output: string, oracle: string): boolean { + return oracle.length > 0 && output.includes(oracle); +} + +function jsonDocuments(raw: string): unknown[] { + try { + const parsed = JSON.parse(raw) as unknown; + return Array.isArray(parsed) ? parsed : [parsed]; + } catch { + return raw.split(/\r?\n/u).flatMap((line) => { + try { + return [JSON.parse(line) as unknown]; + } catch { + return []; + } + }); + } +} + +function record(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +/** Extract only the user-visible final assistant field, never raw tool traces. */ +export function extractFinalAssistantOutput(agent: ToolDisclosureAgent, raw: string): string { + if (agent === "hermes") { + const root = record(jsonDocuments(raw)[0]); + const choice = Array.isArray(root?.choices) ? record(root.choices[0]) : null; + const message = record(choice?.message); + return typeof message?.content === "string" ? message.content : ""; + } + if (agent === "openclaw") { + const parts: string[] = []; + for (const document of jsonDocuments(raw)) { + const root = record(document); + const result = record(root?.result); + const payloads = Array.isArray(result?.payloads) + ? result.payloads + : Array.isArray(root?.payloads) + ? root.payloads + : []; + for (const payload of payloads) { + const text = record(payload)?.text; + if (typeof text === "string") parts.push(text); + } + } + return parts.join("\n"); + } + return ( + raw + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .at(-1) ?? "" + ); +} + +export function buildOpenClawCallLogCommand(options: { + openshellBin?: string; + sandboxName: string; + action: "read" | "reset"; +}): AgentDriverCommand { + validateIdentifier(options.sandboxName, "sandboxName"); + const script = + options.action === "reset" + ? `rm -f -- '${OPENCLAW_BENCH_CALLS_PATH}'` + : `[ ! -f '${OPENCLAW_BENCH_CALLS_PATH}' ] || cat -- '${OPENCLAW_BENCH_CALLS_PATH}'`; + return { + command: options.openshellBin ?? "openshell", + args: ["sandbox", "exec", "-n", options.sandboxName, "--", "sh", "-lc", script], + redactions: [], + }; +} diff --git a/scripts/bench/tool-disclosure/execute.ts b/scripts/bench/tool-disclosure/execute.ts new file mode 100644 index 0000000000..4f0b40431d --- /dev/null +++ b/scripts/bench/tool-disclosure/execute.ts @@ -0,0 +1,624 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; + +import { appendJsonLine, artifactPath, writeJsonArtifact, writeTextArtifact } from "./artifacts"; +import { assembleToolDisclosureRun } from "./assemble-run"; +import { generateCatalogPrefix, type SyntheticCatalog } from "./catalog"; +import { + buildAgentDriverCommand, + buildOpenClawCallLogCommand, + extractFinalAssistantOutput, +} from "./drivers"; +import type { RecordedSyntheticCall } from "./grading"; +import { SyntheticMcpServer } from "./mcp-server"; +import { type QuickTunnel, startQuickTunnel } from "./quick-tunnel"; +import { createToolDisclosureRecordingProxy, type ToolDisclosureRecordingEvent } from "./recorder"; +import type { ScheduledToolDisclosureRun, ToolDisclosureAgent } from "./schedule"; +import { runBoundedCommand } from "./subprocess"; +import type { SyntheticBenchmarkTask, SyntheticTaskSet } from "./tasks"; +import { + countTokensWithVllm, + readVllmProcessStartTime, + readVllmTokenSnapshot, + tokenDelta, +} from "./telemetry"; +import type { ToolDisclosureManifest, ToolDisclosureRun } from "./types"; + +export interface SanitizedRunEvidence { + run_id: string; + recorder_events: readonly ToolDisclosureRecordingEvent[]; + calls: readonly RecordedSyntheticCall[]; + invocation: { exit_code: number | null; timed_out: boolean; elapsed_ms: number }; + initial_schema_tokens: number; + prompt_tokens?: number; + completion_tokens?: number; + final_oracles_present: boolean; + failure_outcome?: "setup-error" | "context-overflow"; +} + +export interface AttemptJournalEntry { + raw: SanitizedRunEvidence; + run: ToolDisclosureRun; +} + +export function recoverAttemptJournal(outputDir: string): AttemptJournalEntry[] { + const file = artifactPath(outputDir, "attempt-journal.jsonl"); + if (!fs.existsSync(file)) return []; + const lines = fs.readFileSync(file, "utf8").split("\n"); + const entries: AttemptJournalEntry[] = []; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + if (!line.trim()) continue; + try { + entries.push(JSON.parse(line) as AttemptJournalEntry); + } catch { + const isTrailing = lines.slice(index + 1).every((candidate) => !candidate.trim()); + if (!isTrailing) throw new Error(`attempt journal is corrupt at line ${index + 1}`); + writeTextArtifact( + outputDir, + "attempt-journal.jsonl", + entries.map((entry) => JSON.stringify(entry)).join("\n") + (entries.length ? "\n" : ""), + ); + break; + } + } + return entries; +} + +export function materializeAttemptJournal( + outputDir: string, + entries: readonly AttemptJournalEntry[], +): void { + const lines = (select: (entry: AttemptJournalEntry) => unknown): string => + entries.map((entry) => JSON.stringify(select(entry))).join("\n") + (entries.length ? "\n" : ""); + writeTextArtifact( + outputDir, + "raw-events.jsonl", + lines((entry) => entry.raw), + ); + writeTextArtifact( + outputDir, + "runs.jsonl", + lines((entry) => entry.run), + ); +} + +export interface LiveCampaignConfiguration { + campaign: 1 | 2; + upstream_vllm_url: string; + telemetry_url: string; + tokenizer_model: string; + recorder_port: number; + managed_inference_base_url: string; + vllm_container_name: string; + vllm_container_id: string; + sandbox_names: Record; + /** Private live sandbox instance IDs, verified against status and published only as hashes. */ + sandbox_instance_ids: Record; + sandbox_container_names: Record; + timeout_ms?: number; + openshell_bin?: string; + nemoclaw_bin?: string; + cloudflared_bin?: string; + docker_bin?: string; +} + +export interface CampaignAttestation { + campaign_id: string; + vllm_process_start_time_seconds: number; + inference_container_id_sha256: string; + inference_config_sha256: string; + inference_image_digest: string; + sandbox_cells: readonly { + cell: string; + instance_id_sha256: string; + status_sha256: string; + image_digest: string; + }[]; +} + +function sha256(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +async function attestVllmContainer(options: { + config: LiveCampaignConfiguration; + manifest: ToolDisclosureManifest; +}): Promise< + Pick< + CampaignAttestation, + "inference_container_id_sha256" | "inference_config_sha256" | "inference_image_digest" + > +> { + const result = await runBoundedCommand({ + command: options.config.docker_bin ?? "docker", + args: ["inspect", options.config.vllm_container_name], + timeoutMs: 30_000, + }); + if (result.exit_code !== 0 || result.timed_out || result.output_truncated) { + throw new Error("vLLM container inspection failed"); + } + const containers = JSON.parse(result.stdout) as Array<{ + Id?: string; + Image?: string; + Config?: { Cmd?: string[]; Entrypoint?: string[]; Env?: string[] }; + }>; + const container = containers[0]; + const expected = options.manifest.inference; + const publicConfig = JSON.stringify({ + cmd: container?.Config?.Cmd ?? [], + entrypoint: container?.Config?.Entrypoint ?? [], + env: (container?.Config?.Env ?? []).filter((value) => + [ + expected.model_id, + expected.model_revision, + expected.tool_call_parser, + expected.reasoning_parser, + ].some((needle) => value.includes(needle)), + ), + }); + if ( + !container || + container.Id !== options.config.vllm_container_id || + container.Image !== expected.container_digest || + ![ + expected.model_id, + expected.model_revision, + expected.tool_call_parser, + expected.reasoning_parser, + ...expected.public_vllm_flags, + ].every((value) => result.stdout.includes(value)) + ) { + throw new Error("live vLLM container does not match the frozen manifest"); + } + return { + inference_container_id_sha256: sha256(container.Id), + inference_config_sha256: sha256(publicConfig), + inference_image_digest: container.Image, + }; +} + +async function attestSandbox(options: { + config: LiveCampaignConfiguration; + manifest: ToolDisclosureManifest; + cell: string; +}): Promise { + const sandboxName = options.config.sandbox_names[options.cell]; + const instanceId = options.config.sandbox_instance_ids[options.cell]; + const imageDigest = options.manifest.environment.sandbox_image_digests?.[options.cell] ?? ""; + if (!instanceId) throw new Error(`live config is missing sandbox instance ID ${options.cell}`); + const result = await runBoundedCommand({ + command: options.config.nemoclaw_bin ?? "nemoclaw", + args: [sandboxName, "status", "--json"], + timeoutMs: 120_000, + }); + if (result.exit_code !== 0 || result.timed_out || result.output_truncated) { + throw new Error(`sandbox status attestation failed for ${options.cell}`); + } + const status = JSON.parse(result.stdout) as { + found?: boolean; + agent?: string; + gatewayState?: string; + }; + const [agent, mode] = options.cell.split(":"); + if (!status.found || status.agent !== agent || status.gatewayState !== "present") { + throw new Error(`sandbox status does not attest the expected cell ${options.cell}`); + } + const containerName = options.config.sandbox_container_names[options.cell]; + if (!containerName) throw new Error(`live config is missing container name ${options.cell}`); + const inspected = await runBoundedCommand({ + command: options.config.docker_bin ?? "docker", + args: ["inspect", containerName], + timeoutMs: 30_000, + }); + if (inspected.exit_code !== 0 || inspected.timed_out || inspected.output_truncated) { + throw new Error(`container inspection failed for ${options.cell}`); + } + const containers = JSON.parse(inspected.stdout) as Array<{ + Id?: string; + Image?: string; + Config?: { Env?: string[] }; + }>; + const container = containers[0]; + if ( + !container || + container.Id !== instanceId || + container.Image !== imageDigest || + !container.Config?.Env?.includes(`NEMOCLAW_TOOL_DISCLOSURE=${mode}`) + ) { + throw new Error(`live container does not attest the expected cell ${options.cell}`); + } + return { + cell: options.cell, + instance_id_sha256: sha256(instanceId), + status_sha256: sha256(`${result.stdout}\n${inspected.stdout}`), + image_digest: imageDigest, + }; +} + +function recordCampaignAttestation(outputDir: string, attestation: CampaignAttestation): void { + const file = artifactPath(outputDir, "campaign-attestations.json"); + const existing = fs.existsSync(file) + ? (JSON.parse(fs.readFileSync(file, "utf8")) as CampaignAttestation[]) + : []; + const priorSame = existing.find((item) => item.campaign_id === attestation.campaign_id); + if (priorSame) { + if (priorSame.vllm_process_start_time_seconds !== attestation.vllm_process_start_time_seconds) { + throw new Error("campaign resume attempted against a different vLLM process"); + } + const priorInstances = priorSame.sandbox_cells.map((cell) => cell.instance_id_sha256).sort(); + const currentInstances = attestation.sandbox_cells + .map((cell) => cell.instance_id_sha256) + .sort(); + if (JSON.stringify(priorInstances) !== JSON.stringify(currentInstances)) { + throw new Error("campaign resume attempted against different sandbox instances"); + } + return; + } + for (const prior of existing) { + if (prior.vllm_process_start_time_seconds === attestation.vllm_process_start_time_seconds) { + throw new Error("campaigns must use distinct fresh vLLM processes"); + } + const priorIds = new Set(prior.sandbox_cells.map((cell) => cell.instance_id_sha256)); + if (attestation.sandbox_cells.some((cell) => priorIds.has(cell.instance_id_sha256))) { + throw new Error("campaigns must use distinct fresh sandbox instances"); + } + } + writeJsonArtifact(outputDir, "campaign-attestations.json", [...existing, attestation]); +} + +function sandboxKey(run: ScheduledToolDisclosureRun): string { + return `${run.agent}:${run.mode}:${run.catalog_size}`; +} + +function parseCalls(raw: string): RecordedSyntheticCall[] { + return raw + .split(/\r?\n/u) + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as RecordedSyntheticCall); +} + +function isContextOverflow(stdout: string, stderr: string): boolean { + return /context(?:_| )length(?:_| )exceeded|maximum context length|context window|too many tokens/iu.test( + `${stdout}\n${stderr}`, + ); +} + +async function command(built: ReturnType, timeoutMs: number) { + return await runBoundedCommand({ + command: built.command, + args: built.args, + timeoutMs, + redactions: built.redactions, + }); +} + +async function configureMcpSandbox(options: { + config: LiveCampaignConfiguration; + agent: ToolDisclosureAgent; + sandboxName: string; + url: string; + token: string; + tokenEnv: string; +}): Promise { + const binary = options.config.nemoclaw_bin ?? "nemoclaw"; + const invoke = async (args: string[]) => { + const result = await runBoundedCommand({ + command: binary, + args, + env: { ...process.env, [options.tokenEnv]: options.token }, + timeoutMs: 20 * 60_000, + redactions: [options.token], + }); + return result; + }; + const run = async (args: string[]): Promise => { + const result = await invoke(args); + if (result.exit_code !== 0 || result.timed_out || result.output_truncated) { + throw new Error(`MCP setup failed for ${options.agent} sandbox`); + } + }; + if (options.agent === "hermes") { + await run([ + options.sandboxName, + "shields", + "down", + "--timeout", + "15m", + "--reason", + "tool disclosure benchmark setup", + ]); + } + try { + const listed = await invoke([options.sandboxName, "mcp", "list", "--json"]); + if (listed.exit_code !== 0 || listed.timed_out || listed.output_truncated) { + throw new Error(`MCP inventory failed for ${options.agent} sandbox`); + } + if (/"(?:name|server)"\s*:\s*"benchmark"/u.test(listed.stdout)) { + await run([options.sandboxName, "mcp", "remove", "benchmark", "--force"]); + } + await run([ + options.sandboxName, + "mcp", + "add", + "benchmark", + "--url", + options.url, + "--env", + options.tokenEnv, + ]); + } finally { + if (options.agent === "hermes") await run([options.sandboxName, "shields", "up"]); + } +} + +function taskMap( + primary: SyntheticTaskSet, + stress: SyntheticTaskSet, +): Map { + return new Map([...primary.tasks, ...stress.tasks].map((task) => [task.id, task] as const)); +} + +/** Execute one frozen campaign against already-created fresh sandboxes and vLLM. */ +export async function executeCampaign(options: { + outputDir: string; + config: LiveCampaignConfiguration; + manifest: ToolDisclosureManifest; + catalog: SyntheticCatalog; + primaryTasks: SyntheticTaskSet; + stressTasks: SyntheticTaskSet; + schedule: readonly ScheduledToolDisclosureRun[]; +}): Promise { + if (options.config.campaign !== 1 && options.config.campaign !== 2) { + throw new Error("live campaign configuration must select campaign 1 or 2"); + } + const scheduled = options.schedule.filter((run) => run.campaign === options.config.campaign); + const expectedKeys = new Set(scheduled.map(sandboxKey)); + for (const key of expectedKeys) { + if (!options.config.sandbox_names[key]) + throw new Error(`live config is missing sandbox ${key}`); + } + const names = [...expectedKeys].map((key) => options.config.sandbox_names[key]); + if (new Set(names).size !== names.length) + throw new Error("every benchmark cell needs a unique sandbox"); + const instanceIds = [...expectedKeys].map((key) => options.config.sandbox_instance_ids[key]); + if (instanceIds.some((value) => !value) || new Set(instanceIds).size !== instanceIds.length) { + throw new Error("every benchmark cell needs a unique live sandbox instance ID"); + } + const containerNames = [...expectedKeys].map( + (key) => options.config.sandbox_container_names[key], + ); + if ( + containerNames.some((value) => !value) || + new Set(containerNames).size !== containerNames.length + ) { + throw new Error("every benchmark cell needs a unique live container name"); + } + if (options.config.tokenizer_model !== options.manifest.inference.model_id) { + throw new Error("tokenizer_model must exactly match manifest.inference.model_id"); + } + if ( + new URL(options.config.upstream_vllm_url).origin !== + new URL(options.config.telemetry_url).origin + ) { + throw new Error("inference and telemetry must address the same vLLM process"); + } + const journal = recoverAttemptJournal(options.outputDir); + const existing = journal.map((entry) => entry.run); + const complete = new Set( + existing.filter((run) => run.outcome !== "setup-error").map((run) => run.run_id), + ); + const token = randomBytes(32).toString("hex"); + const tokenEnv = "NEMOCLAW_BENCHMARK_MCP_BEARER"; + const proxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: options.config.upstream_vllm_url, + port: options.config.recorder_port, + requiredTemperature: 0, + }); + const servers = new Map(); + const tunnels = new Map(); + try { + const proxyAddress = await proxy.start(); + if (proxyAddress.base_url !== options.config.managed_inference_base_url) { + throw new Error("managed inference route does not match the started recorder address"); + } + const processStart = await readVllmProcessStartTime(options.config.telemetry_url); + const inferenceAttestation = await attestVllmContainer({ + config: options.config, + manifest: options.manifest, + }); + const sandboxCells: CampaignAttestation["sandbox_cells"][number][] = []; + for (const cell of [...expectedKeys].sort()) { + sandboxCells.push( + await attestSandbox({ config: options.config, manifest: options.manifest, cell }), + ); + } + recordCampaignAttestation(options.outputDir, { + campaign_id: `campaign-${options.config.campaign}`, + vllm_process_start_time_seconds: processStart, + ...inferenceAttestation, + sandbox_cells: sandboxCells, + }); + for (const size of [16, 64, 256, 512, 2_209]) { + const server = new SyntheticMcpServer(generateCatalogPrefix(options.catalog, size), token); + const address = await server.start(); + servers.set(size, server); + tunnels.set( + size, + await startQuickTunnel({ + port: address.port, + binary: options.config.cloudflared_bin, + }), + ); + } + for (const key of expectedKeys) { + const [agent, , sizeText] = key.split(":") as [ToolDisclosureAgent, string, string]; + if (agent === "openclaw") continue; + await configureMcpSandbox({ + config: options.config, + agent, + sandboxName: options.config.sandbox_names[key], + url: tunnels.get(Number(sizeText))?.mcpUrl ?? "", + token, + tokenEnv, + }); + } + + const tasks = taskMap(options.primaryTasks, options.stressTasks); + const timeoutMs = options.config.timeout_ms ?? 10 * 60_000; + for (const run of scheduled) { + if (complete.has(run.run_id)) continue; + const task = run.task_id ? tasks.get(run.task_id) : undefined; + const sandboxName = options.config.sandbox_names[sandboxKey(run)]; + const mcp = run.agent === "openclaw" ? undefined : servers.get(run.catalog_size); + for ( + let attempt = 0; + attempt <= options.manifest.protocol.retry_setup_failures; + attempt += 1 + ) { + try { + if (run.agent === "openclaw") { + const reset = await command( + buildOpenClawCallLogCommand({ + openshellBin: options.config.openshell_bin, + sandboxName, + action: "reset", + }), + timeoutMs, + ); + if (reset.exit_code !== 0 || reset.timed_out) + throw new Error("OpenClaw call-log reset failed"); + } + mcp?.beginRun(run.run_id); + proxy.beginRun(run.run_id); + const before = await readVllmTokenSnapshot(options.config.telemetry_url); + const invocation = await command( + buildAgentDriverCommand({ + openshellBin: options.config.openshell_bin, + sandboxName, + agent: run.agent, + prompt: task?.prompt ?? "Without using tools, reply exactly STATIC-CAPTURE.", + sessionId: run.run_id, + }), + timeoutMs, + ); + if (invocation.output_truncated) + throw new Error("agent output exceeded the evidence bound"); + const after = await readVllmTokenSnapshot(options.config.telemetry_url); + const recorderEvents = proxy.endRun(); + const snapshots = proxy.consumeToolSchemaSnapshots(run.run_id); + if (!snapshots[0]) throw new Error("recorder did not capture an initial tool schema"); + const initialSchemaTokens = await countTokensWithVllm( + options.config.telemetry_url, + options.config.tokenizer_model, + snapshots[0].canonical_tools_json, + ); + let calls: RecordedSyntheticCall[]; + if (run.agent === "openclaw") { + const callLog = await command( + buildOpenClawCallLogCommand({ + openshellBin: options.config.openshell_bin, + sandboxName, + action: "read", + }), + timeoutMs, + ); + if (callLog.exit_code !== 0 || callLog.timed_out || callLog.output_truncated) { + throw new Error("OpenClaw call-log read failed"); + } + calls = parseCalls(callLog.stdout); + } else { + calls = mcp?.endRun() ?? []; + } + const tokens = tokenDelta(before, after); + if (!tokens.available) throw new Error("vLLM token counters were unavailable or reset"); + const finalOutput = extractFinalAssistantOutput(run.agent, invocation.stdout); + const classifiedFailure = + run.phase !== "static-visibility" && + invocation.exit_code !== 0 && + isContextOverflow(invocation.stdout, invocation.stderr) + ? ("context-overflow" as const) + : undefined; + const rawEvidence: SanitizedRunEvidence = { + run_id: run.run_id, + recorder_events: recorderEvents, + calls, + invocation: { + exit_code: invocation.exit_code, + timed_out: invocation.timed_out, + elapsed_ms: invocation.elapsed_ms, + }, + initial_schema_tokens: initialSchemaTokens, + ...(tokens.available + ? { prompt_tokens: tokens.prompt_tokens, completion_tokens: tokens.generation_tokens } + : {}), + final_oracles_present: + !task || task.expected_final_includes.every((oracle) => finalOutput.includes(oracle)), + ...(classifiedFailure ? { failure_outcome: classifiedFailure } : {}), + }; + const record = assembleToolDisclosureRun({ + manifest: options.manifest, + scheduled: run, + task, + calls, + recorderEvents, + invocation: { + exit_code: invocation.exit_code, + timed_out: invocation.timed_out, + elapsed_ms: invocation.elapsed_ms, + final_output: finalOutput, + }, + initialSchemaTokens, + ...(tokens.available + ? { promptTokens: tokens.prompt_tokens, completionTokens: tokens.generation_tokens } + : {}), + ...(classifiedFailure ? { failureOutcome: classifiedFailure } : {}), + }); + const entry = { raw: rawEvidence, run: record } satisfies AttemptJournalEntry; + appendJsonLine(options.outputDir, "attempt-journal.jsonl", entry); + journal.push(entry); + break; + } catch { + try { + proxy.endRun(); + } catch {} + try { + mcp?.endRun(); + } catch {} + const setupEvidence: SanitizedRunEvidence = { + run_id: run.run_id, + recorder_events: [], + calls: [], + invocation: { exit_code: null, timed_out: false, elapsed_ms: 0 }, + initial_schema_tokens: 0, + final_oracles_present: false, + failure_outcome: "setup-error", + }; + const setupRun = assembleToolDisclosureRun({ + manifest: options.manifest, + scheduled: run, + task, + calls: [], + recorderEvents: [], + invocation: { exit_code: null, timed_out: false, elapsed_ms: 0, final_output: "" }, + initialSchemaTokens: 0, + failureOutcome: "setup-error", + }); + const entry = { raw: setupEvidence, run: setupRun } satisfies AttemptJournalEntry; + appendJsonLine(options.outputDir, "attempt-journal.jsonl", entry); + journal.push(entry); + if (attempt === options.manifest.protocol.retry_setup_failures) + throw new Error(`run ${run.run_id} exhausted setup retries`); + } + } + } + materializeAttemptJournal(options.outputDir, journal); + } finally { + await Promise.allSettled([...tunnels.values()].map((tunnel) => tunnel.close())); + await Promise.allSettled([...servers.values()].map((server) => server.stop())); + await proxy.stop(); + } +} diff --git a/scripts/bench/tool-disclosure/grading.ts b/scripts/bench/tool-disclosure/grading.ts new file mode 100644 index 0000000000..fad0b04c1e --- /dev/null +++ b/scripts/bench/tool-disclosure/grading.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { canonicalJson, sha256Hex } from "./catalog"; +import type { SyntheticBenchmarkTask } from "./tasks"; +import type { BenchmarkRunOutcome, RunCorrectness } from "./types"; + +export interface RecordedSyntheticCall { + tool_name: string; + arguments_sha256: string; + result_nonce: string | null; + success: boolean; +} + +export interface GradedTaskRun { + outcome: Extract; + correctness: RunCorrectness; +} + +/** Grade a run only from deterministic call metadata and the final model output. */ +export function gradeTaskRun( + task: SyntheticBenchmarkTask, + calls: readonly RecordedSyntheticCall[], + finalOutput: string, +): GradedTaskRun { + const expectedNames = task.expected_calls.map((call) => call.tool_name); + const actualNames = calls.map((call) => call.tool_name); + const expectedHashes = task.expected_calls.map((call) => + sha256Hex(canonicalJson(call.arguments)), + ); + const expectedNonces = task.expected_calls.map((call) => call.result_nonce); + const expectedCallCount = calls.length === task.expected_calls.length; + const expectedToolOrder = + expectedCallCount && actualNames.every((name, index) => name === expectedNames[index]); + const expectedArguments = + expectedCallCount && + calls.every( + (call, index) => + call.success && + call.arguments_sha256 === expectedHashes[index] && + call.result_nonce === expectedNonces[index], + ); + const expectedToolNames = + expectedNames.length === actualNames.length && + [...expectedNames].sort().every((name, index) => name === [...actualNames].sort()[index]); + const oraclePresent = + task.expected_final_includes.length > 0 && + task.expected_final_includes.every((oracle) => finalOutput.includes(oracle)); + const unnecessaryToolCalls = calls.filter( + (call, index) => call.tool_name !== expectedNames[index], + ).length; + const taskSuccess = + expectedToolNames && + expectedToolOrder && + expectedArguments && + expectedCallCount && + oraclePresent && + unnecessaryToolCalls === 0; + + return { + outcome: taskSuccess ? "success" : "incorrect", + correctness: { + task_success: taskSuccess, + expected_tool_names: expectedToolNames, + expected_tool_order: expectedToolOrder, + expected_arguments: expectedArguments, + expected_call_count: expectedCallCount, + nonce_present: oraclePresent, + unnecessary_tool_calls: unnecessaryToolCalls, + }, + }; +} diff --git a/scripts/bench/tool-disclosure/mcp-server.ts b/scripts/bench/tool-disclosure/mcp-server.ts new file mode 100644 index 0000000000..14b486f4ba --- /dev/null +++ b/scripts/bench/tool-disclosure/mcp-server.ts @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import http, { type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { + canonicalJson, + executeSyntheticTool, + type JsonObject, + type SyntheticCatalog, + type SyntheticTool, +} from "./catalog"; + +const MAX_BODY_BYTES = 8 * 1024 * 1024; +const SAFE_RUN_ID = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,255}$/u; +const NOTIFICATIONS = new Set([ + "notifications/initialized", + "notifications/cancelled", + "notifications/progress", + "notifications/roots/list_changed", + "notifications/elicitation/complete", +]); + +export interface SyntheticMcpCallEvent { + run_id: string; + sequence: number; + tool_name: string; + arguments_sha256: string; + result_nonce: string | null; + success: boolean; +} + +export interface SyntheticMcpServerAddress { + host: "127.0.0.1"; + port: number; + local_url: string; +} + +function jsonResponse(response: ServerResponse, status: number, value: unknown): void { + const body = JSON.stringify(value); + response.writeHead(status, { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + "cache-control": "no-store", + }); + response.end(body); +} + +async function readBody(request: IncomingMessage): Promise { + return await new Promise((resolve) => { + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + const finish = (value: Buffer | null): void => { + if (settled) return; + settled = true; + resolve(value); + }; + request.on("data", (chunk: Buffer | string) => { + if (settled) return; + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += buffer.length; + if (size > MAX_BODY_BYTES) { + request.resume(); + finish(null); + } else { + chunks.push(buffer); + } + }); + request.once("end", () => finish(Buffer.concat(chunks, size))); + request.once("aborted", () => finish(null)); + request.once("error", () => finish(null)); + }); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function asArguments(value: unknown): JsonObject | null { + if (!isRecord(value)) return null; + return value as JsonObject; +} + +function mcpTool(tool: SyntheticTool): Record { + return { + name: tool.definition.function.name, + description: tool.definition.function.description, + inputSchema: tool.definition.function.parameters, + }; +} + +export class SyntheticMcpServer { + private readonly toolsByName: Map; + private readonly bearerToken: string; + private readonly server: http.Server; + private activeRun: { id: string; sequence: number; eventStartIndex: number } | null = null; + private readonly events: SyntheticMcpCallEvent[] = []; + + constructor(catalog: SyntheticCatalog, bearerToken: string) { + if (!bearerToken || bearerToken.length > 4_096) { + throw new Error("benchmark MCP bearer token must be non-empty and bounded"); + } + this.bearerToken = bearerToken; + this.toolsByName = new Map( + catalog.tools.map((tool) => [tool.definition.function.name, tool] as const), + ); + if (this.toolsByName.size !== catalog.size) { + throw new Error("benchmark MCP catalog contains duplicate tool names"); + } + this.server = http.createServer((request, response) => { + void this.handle(request, response).catch(() => { + jsonResponse(response, 500, { error: "benchmark MCP server failure" }); + }); + }); + this.server.requestTimeout = 120_000; + this.server.headersTimeout = 60_000; + } + + async start(port = 0): Promise { + if (!Number.isInteger(port) || port < 0 || port > 65_535) { + throw new Error("benchmark MCP port must be between 0 and 65535"); + } + await new Promise((resolve, reject) => { + this.server.once("error", reject); + this.server.listen(port, "127.0.0.1", () => resolve()); + }); + const address = this.server.address() as AddressInfo | null; + if (!address || address.address !== "127.0.0.1") { + await this.stop(); + throw new Error("benchmark MCP server failed to bind loopback"); + } + return { + host: "127.0.0.1", + port: address.port, + local_url: `http://127.0.0.1:${address.port}/mcp`, + }; + } + + async stop(): Promise { + await new Promise((resolve) => this.server.close(() => resolve())); + } + + beginRun(runId: string): void { + if (!SAFE_RUN_ID.test(runId)) throw new Error("invalid benchmark MCP run id"); + if (this.activeRun) throw new Error("a benchmark MCP run is already active"); + this.activeRun = { id: runId, sequence: 0, eventStartIndex: this.events.length }; + } + + endRun(): SyntheticMcpCallEvent[] { + const run = this.activeRun; + if (!run) throw new Error("no benchmark MCP run is active"); + this.activeRun = null; + return this.events + .slice(run.eventStartIndex) + .filter((event) => event.run_id === run.id) + .map((event) => ({ ...event })); + } + + private record( + toolName: string, + argumentsValue: JsonObject, + nonce: string | null, + success: boolean, + ): void { + const run = this.activeRun; + if (!run) return; + run.sequence += 1; + this.events.push({ + run_id: run.id, + sequence: run.sequence, + tool_name: toolName, + arguments_sha256: executeArgumentHash(argumentsValue), + result_nonce: nonce, + success, + }); + } + + private async handle(request: IncomingMessage, response: ServerResponse): Promise { + const pathname = new URL(request.url ?? "/", "http://127.0.0.1").pathname; + if (pathname !== "/mcp") { + jsonResponse(response, 404, { error: "not found" }); + return; + } + if (request.method === "HEAD" || request.method === "GET") { + response.writeHead(405, { allow: "POST" }); + response.end(); + return; + } + if (request.method !== "POST") { + jsonResponse(response, 405, { error: "method not allowed" }); + return; + } + if (request.headers.authorization !== `Bearer ${this.bearerToken}`) { + jsonResponse(response, 401, { error: "authentication required" }); + return; + } + const body = await readBody(request); + if (!body) { + jsonResponse(response, 413, { error: "request rejected" }); + return; + } + let payload: unknown; + try { + payload = JSON.parse(body.toString("utf8")); + } catch { + jsonResponse(response, 400, { error: "invalid JSON" }); + return; + } + if (!isRecord(payload) || typeof payload.method !== "string") { + jsonResponse(response, 400, { error: "invalid JSON-RPC request" }); + return; + } + if (NOTIFICATIONS.has(payload.method)) { + response.writeHead(202); + response.end(); + return; + } + const id = payload.id ?? 1; + if (payload.method === "initialize") { + const params = isRecord(payload.params) ? payload.params : {}; + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + result: { + protocolVersion: + typeof params.protocolVersion === "string" ? params.protocolVersion : "2025-03-26", + capabilities: { tools: {} }, + serverInfo: { name: "nemoclaw-tool-disclosure-benchmark", version: "1.0.0" }, + }, + }); + return; + } + if (payload.method === "tools/list") { + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + result: { tools: [...this.toolsByName.values()].map(mcpTool) }, + }); + return; + } + if (payload.method === "tools/call") { + const params = isRecord(payload.params) ? payload.params : {}; + const name = typeof params.name === "string" ? params.name : ""; + const argumentsValue = asArguments(params.arguments); + const tool = this.toolsByName.get(name); + if (!tool || !argumentsValue) { + this.record(name || "[invalid-name]", argumentsValue ?? {}, null, false); + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + error: { code: -32602, message: "invalid benchmark tool call" }, + }); + return; + } + try { + const result = executeSyntheticTool(tool, argumentsValue); + this.record(name, argumentsValue, result.nonce, true); + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + result: { content: [{ type: "text", text: JSON.stringify(result) }], isError: false }, + }); + } catch { + this.record(name, argumentsValue, null, false); + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + error: { code: -32602, message: "invalid benchmark tool arguments" }, + }); + } + return; + } + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + error: { code: -32601, message: "method not found" }, + }); + } +} + +export function executeArgumentHash(argumentsValue: JsonObject): string { + return createHash("sha256").update(canonicalJson(argumentsValue), "utf8").digest("hex"); +} diff --git a/scripts/bench/tool-disclosure/openclaw-fixture.ts b/scripts/bench/tool-disclosure/openclaw-fixture.ts new file mode 100644 index 0000000000..76157bfb3f --- /dev/null +++ b/scripts/bench/tool-disclosure/openclaw-fixture.ts @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import type { SyntheticCatalog } from "./catalog"; + +const SAFE_FIXTURE_DIR = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u; + +export interface OpenClawFixtureOptions { + outputDir: string; + catalog: SyntheticCatalog; + sandboxBase: string; +} + +export interface OpenClawFixtureResult { + directory: string; + files: readonly string[]; +} + +function writeNewFile(directory: string, name: string, content: string): void { + const destination = path.join(directory, name); + if (path.dirname(destination) !== directory) + throw new Error("fixture file escaped output directory"); + fs.writeFileSync(destination, content, { encoding: "utf8", flag: "wx", mode: 0o600 }); +} + +function pluginSource(): string { + return `// Generated by the NemoClaw progressive tool-disclosure benchmark.\n\ +import { createHash } from "node:crypto";\n\ +import fs from "node:fs";\n\ +\n\ +const catalog = JSON.parse(fs.readFileSync(new URL("./catalog.json", import.meta.url), "utf8"));\n\ +const callsPath = process.env.NEMOCLAW_BENCH_CALLS_PATH || "/sandbox/.nemoclaw-benchmark/calls.jsonl";\n\ +\n\ +function canonical(value) {\n\ + if (Array.isArray(value)) return "[" + value.map(canonical).join(",") + "]";\n\ + if (value && typeof value === "object") {\n\ + return "{" + Object.keys(value).sort().map((key) => JSON.stringify(key) + ":" + canonical(value[key])).join(",") + "}";\n\ + }\n\ + return JSON.stringify(value);\n\ +}\n\ +\n\ +function sha256(value) {\n\ + return createHash("sha256").update(value, "utf8").digest("hex");\n\ +}\n\ +\n\ +function execute(tool, args) {\n\ + const argumentsSha256 = sha256(canonical(args));\n\ + const nonce = "nonce_" + sha256(tool.handler.key + "\\0" + canonical(args)).slice(0, 20);\n\ + const result = {\n\ + ok: true,\n\ + tool_name: tool.definition.function.name,\n\ + category: tool.category,\n\ + operation: tool.operation,\n\ + nonce,\n\ + arguments_sha256: argumentsSha256,\n\ + summary: tool.operation.replaceAll("_", " ") + " completed for " + String(args.resource_id),\n\ + };\n\ + fs.appendFileSync(callsPath, JSON.stringify({\n\ + tool_name: result.tool_name,\n\ + arguments_sha256: argumentsSha256,\n\ + result_nonce: nonce,\n\ + success: true,\n\ + }) + "\\n", { encoding: "utf8", mode: 0o660 });\n\ + return result;\n\ +}\n\ +\n\ +export default function registerBenchmarkTools(api) {\n\ + if (api.registrationMode && api.registrationMode !== "full") return;\n\ + for (const tool of catalog.tools) {\n\ + const definition = tool.definition.function;\n\ + api.registerTool({\n\ + name: definition.name,\n\ + label: definition.name,\n\ + description: definition.description,\n\ + parameters: definition.parameters,\n\ + async execute(_toolCallId, params) {\n\ + const result = execute(tool, params);\n\ + return {\n\ + content: [{ type: "text", text: JSON.stringify(result) }],\n\ + details: result,\n\ + };\n\ + },\n\ + });\n\ + }\n\ +}\n`; +} + +function dockerfile(sandboxBase: string): string { + return `ARG SANDBOX_BASE=${sandboxBase}\n\ +FROM \${SANDBOX_BASE}\n\ +\n\ +ARG NEMOCLAW_TOOL_DISCLOSURE=progressive\n\ +ENV NEMOCLAW_TOOL_DISCLOSURE=\${NEMOCLAW_TOOL_DISCLOSURE}\n\ +\n\ +COPY plugin/ /opt/nemoclaw-tool-disclosure-benchmark/\n\ +RUN mkdir -p /sandbox/.openclaw/extensions \\\n && cp -a /opt/nemoclaw-tool-disclosure-benchmark /sandbox/.openclaw/extensions/nemoclaw-tool-disclosure-benchmark \\\n && install -d -m 2770 -o sandbox -g sandbox /sandbox/.nemoclaw-benchmark \\\n && openclaw doctor --fix\n\ +\n\ +WORKDIR /opt/nemoclaw\n`; +} + +/** Generate a self-contained custom-image context for an OpenClaw catalog size. */ +export function writeOpenClawFixture(options: OpenClawFixtureOptions): OpenClawFixtureResult { + const root = path.resolve(options.outputDir); + if (fs.existsSync(root)) { + const stat = fs.lstatSync(root); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error("OpenClaw fixture output must be a real directory"); + } + if (fs.readdirSync(root).length > 0) throw new Error("OpenClaw fixture output must be empty"); + } else { + fs.mkdirSync(root, { recursive: true, mode: 0o700 }); + } + if (!SAFE_FIXTURE_DIR.test(path.basename(root))) { + throw new Error("OpenClaw fixture directory has an unsafe name"); + } + + const pluginDir = path.join(root, "plugin"); + fs.mkdirSync(pluginDir, { mode: 0o700 }); + const names = options.catalog.tools.map((tool) => tool.definition.function.name); + writeNewFile(pluginDir, "catalog.json", `${JSON.stringify(options.catalog, null, 2)}\n`); + writeNewFile(pluginDir, "index.js", pluginSource()); + writeNewFile( + pluginDir, + "package.json", + `${JSON.stringify( + { + name: "nemoclaw-tool-disclosure-benchmark", + version: "1.0.0", + private: true, + type: "module", + openclaw: { extensions: ["./index.js"] }, + }, + null, + 2, + )}\n`, + ); + writeNewFile( + pluginDir, + "openclaw.plugin.json", + `${JSON.stringify( + { + id: "nemoclaw-tool-disclosure-benchmark", + name: "NemoClaw Tool Disclosure Benchmark", + version: "1.0.0", + description: "Deterministic synthetic tools for progressive disclosure benchmarks.", + contracts: { tools: names }, + configSchema: { type: "object", properties: {}, additionalProperties: false }, + }, + null, + 2, + )}\n`, + ); + writeNewFile(root, "Dockerfile", dockerfile(options.sandboxBase)); + writeNewFile(root, ".dockerignore", "*\n!Dockerfile\n!plugin\n!plugin/**\n"); + + return { + directory: root, + files: [ + ".dockerignore", + "Dockerfile", + "plugin/catalog.json", + "plugin/index.js", + "plugin/openclaw.plugin.json", + "plugin/package.json", + ], + }; +} diff --git a/scripts/bench/tool-disclosure/quick-tunnel.ts b/scripts/bench/tool-disclosure/quick-tunnel.ts new file mode 100644 index 0000000000..b05e60374c --- /dev/null +++ b/scripts/bench/tool-disclosure/quick-tunnel.ts @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, spawn } from "node:child_process"; + +const ORIGIN_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com(?=$|[\s"'\\/])/iu; +const SAFE_ENV = new Set([ + "PATH", + "TMPDIR", + "TMP", + "TEMP", + "LANG", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +]); + +export interface QuickTunnel { + origin: string; + mcpUrl: string; + close(): Promise; +} + +export function parseQuickTunnelOrigin(text: string): string | null { + return text.match(ORIGIN_PATTERN)?.[0] ?? null; +} + +export function buildQuickTunnelArgs(port: number): string[] { + if (!Number.isInteger(port) || port < 1 || port > 65_535) { + throw new Error("quick-tunnel origin port must be between 1 and 65535"); + } + return [ + "tunnel", + "--no-autoupdate", + "--protocol", + "http2", + "--url", + `http://127.0.0.1:${port}`, + "--loglevel", + "info", + ]; +} + +function subprocessEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const selected: NodeJS.ProcessEnv = {}; + for (const [name, value] of Object.entries(env)) { + if (value !== undefined && (SAFE_ENV.has(name) || name.startsWith("LC_"))) { + selected[name] = value; + } + } + return selected; +} + +function stopChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); + return new Promise((resolve) => { + const timer = setTimeout(() => { + try { + child.kill("SIGKILL"); + } catch { + // The process exited between the state check and signal. + } + }, 5_000); + timer.unref(); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + try { + child.kill("SIGTERM"); + } catch { + clearTimeout(timer); + resolve(); + } + }); +} + +async function probe(origin: string, fetchImpl: typeof fetch): Promise { + try { + const response = await fetchImpl(`${origin}/mcp`, { + method: "HEAD", + redirect: "error", + signal: AbortSignal.timeout(5_000), + }); + await response.body?.cancel(); + return response.status === 405; + } catch { + return false; + } +} + +export async function startQuickTunnel(options: { + port: number; + binary?: string; + timeoutMs?: number; + fetchImpl?: typeof fetch; + env?: NodeJS.ProcessEnv; +}): Promise { + const child = spawn(options.binary ?? "cloudflared", buildQuickTunnelArgs(options.port), { + detached: false, + env: subprocessEnv(options.env ?? process.env), + stdio: ["ignore", "pipe", "pipe"], + }); + const timeoutMs = options.timeoutMs ?? 45_000; + const fetchImpl = options.fetchImpl ?? fetch; + let origin: string | null = null; + let carry = ""; + let spawnError: Error | undefined; + const inspect = (chunk: Buffer | string): void => { + const candidate = `${carry}${chunk.toString()}`; + origin ??= parseQuickTunnelOrigin(candidate); + carry = candidate.slice(-512); + }; + child.stdout?.on("data", inspect); + child.stderr?.on("data", inspect); + child.once("error", (error) => { + spawnError = error; + }); + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (spawnError) break; + if (child.exitCode !== null || child.signalCode !== null) break; + if (origin && (await probe(origin, fetchImpl))) { + const published = origin; + return { + origin: published, + mcpUrl: `${published}/mcp`, + close: () => stopChild(child), + }; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + await stopChild(child); + if (spawnError) throw new Error(`cloudflared quick tunnel failed to start (${spawnError.name})`); + throw new Error("cloudflared quick tunnel did not become ready before the timeout"); +} diff --git a/scripts/bench/tool-disclosure/recorder.ts b/scripts/bench/tool-disclosure/recorder.ts new file mode 100644 index 0000000000..5713334340 --- /dev/null +++ b/scripts/bench/tool-disclosure/recorder.ts @@ -0,0 +1,838 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomUUID } from "node:crypto"; +import http, { + type IncomingHttpHeaders, + type IncomingMessage, + type OutgoingHttpHeaders, + type Server, + type ServerResponse, +} from "node:http"; +import https from "node:https"; +import { isIP, type Socket } from "node:net"; +import { performance } from "node:perf_hooks"; + +import { canonicalJson, type JsonValue, sha256Hex } from "./catalog"; + +const LOOPBACK_HOST = "127.0.0.1"; +const DEFAULT_MAX_REQUEST_BODY_BYTES = 64 * 1024 * 1024; +const DEFAULT_REQUEST_TIMEOUT_MS = 120_000; +const SAFE_RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/; +const SAFE_TOOL_NAME = /^[A-Za-z0-9_-]{1,128}$/; + +const HOP_BY_HOP_HEADERS = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", +]); + +export type RecorderEndpoint = + | "chat-completions" + | "responses" + | "completions" + | "embeddings" + | "tokenize" + | "models" + | "other-v1"; + +export type RecorderMethod = "GET" | "POST" | "OTHER"; + +export type RecorderOutcome = + | "completed" + | "request-rejected" + | "upstream-timeout" + | "upstream-connection-error" + | "upstream-response-error" + | "client-aborted"; + +export type RecorderErrorReason = + | "request-body-too-large" + | "request-timeout" + | "client-request-error" + | "upstream-timeout" + | "upstream-connection-error" + | "upstream-response-error" + | "proxy-failure" + | "client-aborted"; + +/** + * A deliberately content-free record of one proxied /v1 request. + * + * Do not add URLs, headers, request/response bodies, model identifiers, tool + * descriptions, or thrown error messages to this interface. This object is + * suitable for direct inclusion in the public benchmark evidence bundle. + */ +export interface ToolDisclosureRecordingEvent { + run_id: string; + request_sequence: number; + model_call_sequence: number | null; + endpoint: RecorderEndpoint; + method: RecorderMethod; + visible_tool_count: number; + canonical_tools_json_bytes: number; + tools_sha256: string | null; + tool_names: readonly string[]; + streaming: boolean | null; + status_code: number | null; + started_monotonic_ms: number; + first_byte_monotonic_ms: number | null; + ended_monotonic_ms: number; + duration_ms: number; + time_to_first_byte_ms: number | null; + outcome: RecorderOutcome; + error_reason: RecorderErrorReason | null; +} + +export interface RecordingProxyOptions { + upstreamBaseUrl: string; + /** Must be exactly 127.0.0.1. Exposed so unsafe configuration fails closed. */ + listenHost?: string; + /** Zero selects an ephemeral port. */ + port?: number; + maxRequestBodyBytes?: number; + requestTimeoutMs?: number; + requiredTemperature?: number; +} + +export interface RecordingProxyAddress { + host: typeof LOOPBACK_HOST; + port: number; + base_url: string; +} + +/** + * Ephemeral schema material for exact tokenizer counting. This type must never + * be serialized into the public evidence bundle. + */ +export interface EphemeralToolSchemaSnapshot { + run_id: string; + model_call_sequence: number; + canonical_tools_json: string; +} + +interface ToolMetrics { + visibleToolCount: number; + canonicalToolsJsonBytes: number; + toolsSha256: string | null; + toolNames: readonly string[]; + canonicalToolsJson: string | null; + streaming: boolean | null; + temperature: number | null; +} + +interface PendingRecording { + runId: string; + requestSequence: number; + modelCallSequence: number | null; + endpoint: RecorderEndpoint; + method: RecorderMethod; + startedAt: number; + metrics: ToolMetrics; + finalized: boolean; +} + +interface ActiveRun { + id: string; + requestSequence: number; + modelCallSequence: number; + pendingRequests: number; + eventStartIndex: number; +} + +interface ForwardContext { + bodyComplete: boolean; + cancelled: boolean; + timedOut: boolean; + upstreamRequest: http.ClientRequest | null; +} + +interface FinalizeOptions { + statusCode: number | null; + firstByteAt?: number; + outcome: RecorderOutcome; + errorReason?: RecorderErrorReason; +} + +type BodyReadResult = + | { kind: "body"; body: Buffer } + | { kind: "too-large" } + | { kind: "client-error" }; + +function validatePositiveInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${label} must be a positive integer`); + } +} + +function parseUpstreamBaseUrl(value: string): URL { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error("upstreamBaseUrl must be a valid HTTP(S) URL"); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("upstreamBaseUrl must use HTTP or HTTPS"); + } + const hostname = url.hostname.replace(/^\[/u, "").replace(/\]$/u, "").toLowerCase(); + const loopback = + hostname === "localhost" || + hostname === "::1" || + (isIP(hostname) === 4 && hostname.startsWith("127.")); + if (url.protocol === "http:" && !loopback) { + throw new Error("plaintext upstreamBaseUrl is allowed only on loopback"); + } + if (url.username || url.password) { + throw new Error("upstreamBaseUrl must not contain credentials"); + } + // Query values are never recorded, but accepting them in long-lived + // configuration makes it too easy to embed an API key in a report command. + if (url.search || url.hash) { + throw new Error("upstreamBaseUrl must not contain a query or fragment"); + } + return url; +} + +function classifyEndpoint(pathname: string): RecorderEndpoint { + const normalized = pathname.replace(/\/+$/, "") || "/"; + if (normalized === "/v1/chat/completions") return "chat-completions"; + if (normalized === "/v1/responses") return "responses"; + if (normalized === "/v1/completions") return "completions"; + if (normalized === "/v1/embeddings") return "embeddings"; + if (normalized === "/v1/tokenize") return "tokenize"; + if (normalized === "/v1/models") return "models"; + return "other-v1"; +} + +function classifyMethod(method: string | undefined): RecorderMethod { + if (method === "GET") return "GET"; + if (method === "POST") return "POST"; + return "OTHER"; +} + +function isModelCall(endpoint: RecorderEndpoint, method: RecorderMethod): boolean { + return ( + method === "POST" && + ["chat-completions", "responses", "completions", "embeddings"].includes(endpoint) + ); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function safeToolName(tool: unknown): string | null { + if (!isRecord(tool)) return null; + let candidate: unknown; + if (isRecord(tool.function)) candidate = tool.function.name; + else candidate = tool.name ?? tool.type; + if (typeof candidate !== "string") return null; + return SAFE_TOOL_NAME.test(candidate) ? candidate : "[invalid-name]"; +} + +function inspectTools(body: Buffer): ToolMetrics { + const empty: ToolMetrics = { + visibleToolCount: 0, + canonicalToolsJsonBytes: 0, + toolsSha256: null, + toolNames: [], + canonicalToolsJson: null, + streaming: null, + temperature: null, + }; + if (body.length === 0) return empty; + + try { + const payload: unknown = JSON.parse(body.toString("utf8")); + if (!isRecord(payload)) return empty; + empty.temperature = typeof payload.temperature === "number" ? payload.temperature : null; + if (!Array.isArray(payload.tools)) return empty; + const canonicalTools = canonicalJson(payload.tools as JsonValue); + return { + visibleToolCount: payload.tools.length, + canonicalToolsJsonBytes: Buffer.byteLength(canonicalTools), + toolsSha256: sha256Hex(canonicalTools), + toolNames: payload.tools.flatMap((tool) => { + const name = safeToolName(tool); + return name === null ? [] : [name]; + }), + canonicalToolsJson: canonicalTools, + streaming: typeof payload.stream === "boolean" ? payload.stream : null, + temperature: typeof payload.temperature === "number" ? payload.temperature : null, + }; + } catch { + // Invalid JSON and pathological nesting are forwarded unchanged, but no + // body-derived diagnostic is retained in the public event stream. + return empty; + } +} + +function connectionHeaderNames(headers: IncomingHttpHeaders): Set { + const value = headers.connection; + const joined = Array.isArray(value) ? value.join(",") : (value ?? ""); + return new Set( + joined + .split(",") + .map((item) => item.trim().toLowerCase()) + .filter(Boolean), + ); +} + +function forwardedRequestHeaders( + headers: IncomingHttpHeaders, + bodyLength: number, +): OutgoingHttpHeaders { + const result: OutgoingHttpHeaders = {}; + const connectionNames = connectionHeaderNames(headers); + for (const [name, value] of Object.entries(headers)) { + const normalized = name.toLowerCase(); + if ( + value === undefined || + normalized === "host" || + normalized === "content-length" || + HOP_BY_HOP_HEADERS.has(normalized) || + connectionNames.has(normalized) + ) { + continue; + } + result[name] = value; + } + result["content-length"] = String(bodyLength); + return result; +} + +function forwardedResponseHeaders(headers: IncomingHttpHeaders): OutgoingHttpHeaders { + const result: OutgoingHttpHeaders = {}; + const connectionNames = connectionHeaderNames(headers); + for (const [name, value] of Object.entries(headers)) { + const normalized = name.toLowerCase(); + if ( + value === undefined || + HOP_BY_HOP_HEADERS.has(normalized) || + connectionNames.has(normalized) + ) { + continue; + } + result[name] = value; + } + return result; +} + +function fixedResponse(response: ServerResponse, status: number, error: string): void { + if (response.headersSent || response.destroyed) return; + const body = JSON.stringify({ error }); + response.writeHead(status, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + "cache-control": "no-store", + }); + response.end(body); +} + +function readBoundedBody(request: IncomingMessage, maxBytes: number): Promise { + return new Promise((resolve) => { + let settled = false; + let size = 0; + let chunks: Buffer[] = []; + + const finish = (result: BodyReadResult): void => { + if (settled) return; + settled = true; + chunks = []; + resolve(result); + }; + + request.on("data", (chunk: Buffer | string) => { + if (settled) return; + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += buffer.length; + if (size > maxBytes) { + request.resume(); + finish({ kind: "too-large" }); + return; + } + chunks.push(buffer); + }); + request.once("end", () => { + if (!settled) finish({ kind: "body", body: Buffer.concat(chunks, size) }); + }); + request.once("aborted", () => finish({ kind: "client-error" })); + request.once("error", () => finish({ kind: "client-error" })); + }); +} + +/** + * Loopback-only OpenAI-compatible recording proxy for benchmark measurements. + * The proxy never logs and keeps only content-free events in memory. + */ +export class ToolDisclosureRecordingProxy { + private readonly upstream: URL; + private readonly listenHost: typeof LOOPBACK_HOST; + private readonly configuredPort: number; + private readonly maxRequestBodyBytes: number; + private readonly requestTimeoutMs: number; + private readonly requiredTemperature: number | undefined; + private readonly clock: () => number; + private readonly sockets = new Set(); + private readonly events: ToolDisclosureRecordingEvent[] = []; + private readonly ephemeralSchemas = new Map(); + private server: Server | null = null; + private activeRun: ActiveRun | null = null; + + constructor(options: RecordingProxyOptions) { + const listenHost = options.listenHost ?? LOOPBACK_HOST; + if (listenHost !== LOOPBACK_HOST) { + throw new Error(`recording proxy listenHost must be exactly ${LOOPBACK_HOST}`); + } + const port = options.port ?? 0; + if (!Number.isSafeInteger(port) || port < 0 || port > 65_535) { + throw new Error("recording proxy port must be an integer from 0 through 65535"); + } + const maxRequestBodyBytes = options.maxRequestBodyBytes ?? DEFAULT_MAX_REQUEST_BODY_BYTES; + const requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + validatePositiveInteger(maxRequestBodyBytes, "maxRequestBodyBytes"); + validatePositiveInteger(requestTimeoutMs, "requestTimeoutMs"); + + this.upstream = parseUpstreamBaseUrl(options.upstreamBaseUrl); + this.listenHost = listenHost; + this.configuredPort = port; + this.maxRequestBodyBytes = maxRequestBodyBytes; + this.requestTimeoutMs = requestTimeoutMs; + this.requiredTemperature = options.requiredTemperature; + this.clock = () => performance.now(); + } + + async start(): Promise { + if (this.server) throw new Error("recording proxy is already started"); + + const server = http.createServer((request, response) => { + void this.handleRequest(request, response).catch(() => { + // Keep thrown implementation errors content-free at the network and + // evidence boundaries. The benchmark runner receives a fixed class. + fixedResponse(response, 502, "recording proxy failure"); + }); + }); + server.requestTimeout = this.requestTimeoutMs; + server.headersTimeout = Math.min(this.requestTimeoutMs, 60_000); + server.maxHeadersCount = 200; + server.on("connection", (socket) => { + this.sockets.add(socket); + socket.once("close", () => this.sockets.delete(socket)); + }); + + await new Promise((resolve, reject) => { + const onError = (error: Error): void => { + server.off("listening", onListening); + reject(error); + }; + const onListening = (): void => { + server.off("error", onError); + resolve(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen(this.configuredPort, this.listenHost); + }); + this.server = server; + + const address = server.address(); + if (!address || typeof address === "string" || address.address !== LOOPBACK_HOST) { + await this.stop(); + throw new Error("recording proxy failed to bind the required loopback address"); + } + return { + host: LOOPBACK_HOST, + port: address.port, + base_url: `http://${LOOPBACK_HOST}:${address.port}`, + }; + } + + async stop(): Promise { + const server = this.server; + if (!server) return; + this.server = null; + for (const socket of this.sockets) socket.destroy(); + this.sockets.clear(); + await new Promise((resolve) => server.close(() => resolve())); + } + + beginRun(runId: string = randomUUID()): string { + if (this.activeRun) throw new Error("a recording run is already active"); + if (!SAFE_RUN_ID.test(runId)) { + throw new Error( + "recording run ID must be 1-256 public-safe letters, digits, dots, underscores, colons, or hyphens", + ); + } + this.activeRun = { + id: runId, + requestSequence: 0, + modelCallSequence: 0, + pendingRequests: 0, + eventStartIndex: this.events.length, + }; + return runId; + } + + endRun(): readonly ToolDisclosureRecordingEvent[] { + const run = this.activeRun; + if (!run) throw new Error("no recording run is active"); + if (run.pendingRequests !== 0) { + throw new Error("cannot end a recording run while requests are in flight"); + } + this.activeRun = null; + return this.events + .slice(run.eventStartIndex) + .filter((event) => event.run_id === run.id) + .map((event) => ({ ...event, tool_names: [...event.tool_names] })); + } + + getEvents(runId?: string): readonly ToolDisclosureRecordingEvent[] { + return this.events + .filter((event) => runId === undefined || event.run_id === runId) + .map((event) => ({ ...event, tool_names: [...event.tool_names] })); + } + + resetEvents(): void { + if (this.activeRun) throw new Error("cannot reset events while a recording run is active"); + this.events.length = 0; + this.ephemeralSchemas.clear(); + } + + /** Return and erase private schema content after a run so it can be tokenized exactly. */ + consumeToolSchemaSnapshots(runId: string): readonly EphemeralToolSchemaSnapshot[] { + if (this.activeRun?.id === runId) + throw new Error("cannot consume schemas while a run is active"); + const snapshots = [...this.ephemeralSchemas.values()] + .filter((snapshot) => snapshot.run_id === runId) + .sort((left, right) => left.model_call_sequence - right.model_call_sequence) + .map((snapshot) => ({ ...snapshot })); + for (const snapshot of snapshots) { + this.ephemeralSchemas.delete(`${snapshot.run_id}:${snapshot.model_call_sequence}`); + } + return snapshots; + } + + private reserveRecording( + endpoint: RecorderEndpoint, + method: RecorderMethod, + ): PendingRecording | null { + const run = this.activeRun; + if (!run) return null; + run.requestSequence += 1; + run.pendingRequests += 1; + let modelCallSequence: number | null = null; + if (isModelCall(endpoint, method)) { + run.modelCallSequence += 1; + modelCallSequence = run.modelCallSequence; + } + return { + runId: run.id, + requestSequence: run.requestSequence, + modelCallSequence, + endpoint, + method, + startedAt: this.clock(), + metrics: { + visibleToolCount: 0, + canonicalToolsJsonBytes: 0, + toolsSha256: null, + toolNames: [], + canonicalToolsJson: null, + streaming: null, + temperature: null, + }, + finalized: false, + }; + } + + private finalizeRecording(pending: PendingRecording | null, options: FinalizeOptions): void { + if (!pending || pending.finalized) return; + pending.finalized = true; + const endedAt = this.clock(); + const firstByteAt = options.firstByteAt ?? null; + const event: ToolDisclosureRecordingEvent = { + run_id: pending.runId, + request_sequence: pending.requestSequence, + model_call_sequence: pending.modelCallSequence, + endpoint: pending.endpoint, + method: pending.method, + visible_tool_count: pending.metrics.visibleToolCount, + canonical_tools_json_bytes: pending.metrics.canonicalToolsJsonBytes, + tools_sha256: pending.metrics.toolsSha256, + tool_names: [...pending.metrics.toolNames], + streaming: pending.metrics.streaming, + status_code: options.statusCode, + started_monotonic_ms: pending.startedAt, + first_byte_monotonic_ms: firstByteAt, + ended_monotonic_ms: endedAt, + duration_ms: endedAt - pending.startedAt, + time_to_first_byte_ms: firstByteAt === null ? null : firstByteAt - pending.startedAt, + outcome: options.outcome, + error_reason: options.errorReason ?? null, + }; + this.events.push(Object.freeze(event)); + if (pending.modelCallSequence !== null && pending.metrics.canonicalToolsJson !== null) { + const snapshot: EphemeralToolSchemaSnapshot = { + run_id: pending.runId, + model_call_sequence: pending.modelCallSequence, + canonical_tools_json: pending.metrics.canonicalToolsJson, + }; + this.ephemeralSchemas.set( + `${snapshot.run_id}:${snapshot.model_call_sequence}`, + Object.freeze(snapshot), + ); + } + if (this.activeRun?.id === pending.runId) { + this.activeRun.pendingRequests -= 1; + } + } + + private buildUpstreamUrl(incoming: URL): URL { + const target = new URL(this.upstream.toString()); + const basePath = target.pathname.replace(/\/+$/, ""); + const prefix = basePath.endsWith("/v1") ? basePath.slice(0, -3) : basePath; + target.pathname = `${prefix}${incoming.pathname}` || "/"; + target.search = incoming.search; + return target; + } + + private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise { + let incoming: URL; + try { + incoming = new URL(request.url ?? "/", "http://127.0.0.1"); + } catch { + fixedResponse(response, 400, "invalid request target"); + return; + } + if (incoming.pathname !== "/v1" && !incoming.pathname.startsWith("/v1/")) { + fixedResponse(response, 404, "not found"); + return; + } + + const endpoint = classifyEndpoint(incoming.pathname); + const method = classifyMethod(request.method); + const pending = this.reserveRecording(endpoint, method); + const context: ForwardContext = { + bodyComplete: false, + cancelled: false, + timedOut: false, + upstreamRequest: null, + }; + + const declaredLength = Number(request.headers["content-length"]); + if (Number.isFinite(declaredLength) && declaredLength > this.maxRequestBodyBytes) { + request.resume(); + fixedResponse(response, 413, "request body too large"); + this.finalizeRecording(pending, { + statusCode: 413, + outcome: "request-rejected", + errorReason: "request-body-too-large", + }); + return; + } + + const timer = setTimeout(() => { + context.cancelled = true; + context.timedOut = true; + if (context.upstreamRequest) { + context.upstreamRequest.destroy(); + return; + } + const statusCode = context.bodyComplete ? 504 : 408; + fixedResponse( + response, + statusCode, + context.bodyComplete ? "upstream timeout" : "request timeout", + ); + this.finalizeRecording(pending, { + statusCode, + outcome: context.bodyComplete ? "upstream-timeout" : "request-rejected", + errorReason: context.bodyComplete ? "upstream-timeout" : "request-timeout", + }); + request.destroy(); + }, this.requestTimeoutMs); + timer.unref(); + + try { + const bodyResult = await readBoundedBody(request, this.maxRequestBodyBytes); + context.bodyComplete = true; + if (context.cancelled) return; + if (bodyResult.kind === "too-large") { + fixedResponse(response, 413, "request body too large"); + this.finalizeRecording(pending, { + statusCode: 413, + outcome: "request-rejected", + errorReason: "request-body-too-large", + }); + return; + } + if (bodyResult.kind === "client-error") { + this.finalizeRecording(pending, { + statusCode: null, + outcome: "client-aborted", + errorReason: "client-request-error", + }); + return; + } + + let forwardedBody = bodyResult.body; + if (pending) pending.metrics = inspectTools(forwardedBody); + if ( + pending?.modelCallSequence != null && + this.requiredTemperature !== undefined && + pending.metrics.temperature !== null && + pending.metrics.temperature !== this.requiredTemperature + ) { + fixedResponse(response, 400, "benchmark temperature mismatch"); + this.finalizeRecording(pending, { + statusCode: 400, + outcome: "request-rejected", + errorReason: "proxy-failure", + }); + return; + } + if ( + pending?.modelCallSequence != null && + this.requiredTemperature !== undefined && + pending.metrics.temperature === null + ) { + const payload = JSON.parse(forwardedBody.toString("utf8")) as Record; + payload.temperature = this.requiredTemperature; + forwardedBody = Buffer.from(JSON.stringify(payload), "utf8"); + } + await this.forwardRequest( + request, + response, + this.buildUpstreamUrl(incoming), + forwardedBody, + pending, + context, + ); + } catch { + fixedResponse(response, 502, "recording proxy failure"); + this.finalizeRecording(pending, { + statusCode: response.headersSent ? response.statusCode : 502, + outcome: "request-rejected", + errorReason: "proxy-failure", + }); + } finally { + clearTimeout(timer); + } + } + + private forwardRequest( + incomingRequest: IncomingMessage, + response: ServerResponse, + target: URL, + body: Buffer, + pending: PendingRecording | null, + context: ForwardContext, + ): Promise { + return new Promise((resolve) => { + let settled = false; + let statusCode: number | null = null; + let firstByteAt: number | undefined; + + const finish = (options: FinalizeOptions): void => { + if (settled) return; + settled = true; + this.finalizeRecording(pending, options); + resolve(); + }; + + const transport = target.protocol === "https:" ? https : http; + const upstreamRequest = transport.request( + target, + { + method: incomingRequest.method, + headers: forwardedRequestHeaders(incomingRequest.headers, body.length), + // Native request does not follow redirects. A 3xx is passed to the + // benchmark client with its body and Location semantics intact. + }, + (upstreamResponse) => { + statusCode = upstreamResponse.statusCode ?? 502; + if (!response.headersSent && !response.destroyed) { + response.writeHead(statusCode, forwardedResponseHeaders(upstreamResponse.headers)); + } + + upstreamResponse.once("data", () => { + firstByteAt = this.clock(); + }); + upstreamResponse.once("end", () => { + finish({ + statusCode, + firstByteAt, + outcome: "completed", + }); + }); + upstreamResponse.once("aborted", () => { + finish({ + statusCode, + firstByteAt, + outcome: context.timedOut ? "upstream-timeout" : "upstream-response-error", + errorReason: context.timedOut ? "upstream-timeout" : "upstream-response-error", + }); + if (!response.writableEnded) response.destroy(); + }); + upstreamResponse.once("error", () => { + finish({ + statusCode, + firstByteAt, + outcome: context.timedOut ? "upstream-timeout" : "upstream-response-error", + errorReason: context.timedOut ? "upstream-timeout" : "upstream-response-error", + }); + if (!response.writableEnded) response.destroy(); + }); + upstreamResponse.pipe(response); + }, + ); + context.upstreamRequest = upstreamRequest; + + response.once("close", () => { + if (settled || response.writableEnded) return; + upstreamRequest.destroy(); + finish({ + statusCode, + firstByteAt, + outcome: "client-aborted", + errorReason: "client-aborted", + }); + }); + upstreamRequest.once("error", () => { + if (context.timedOut) { + const recordedStatus = response.headersSent ? statusCode : 504; + fixedResponse(response, 504, "upstream timeout"); + finish({ + statusCode: recordedStatus, + firstByteAt, + outcome: "upstream-timeout", + errorReason: "upstream-timeout", + }); + return; + } + const recordedStatus = response.headersSent ? statusCode : 502; + fixedResponse(response, 502, "upstream unavailable"); + finish({ + statusCode: recordedStatus, + firstByteAt, + outcome: "upstream-connection-error", + errorReason: "upstream-connection-error", + }); + }); + upstreamRequest.end(body); + }); + } +} + +export function createToolDisclosureRecordingProxy( + options: RecordingProxyOptions, +): ToolDisclosureRecordingProxy { + return new ToolDisclosureRecordingProxy(options); +} diff --git a/scripts/bench/tool-disclosure/report.ts b/scripts/bench/tool-disclosure/report.ts new file mode 100644 index 0000000000..774e7fb0e9 --- /dev/null +++ b/scripts/bench/tool-disclosure/report.ts @@ -0,0 +1,252 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildConservativeClaims } from "./statistics"; +import { + type ConfidenceInterval, + type EvidenceArtifact, + TOOL_DISCLOSURE_SCHEMA_VERSION, + type ToolDisclosureManifest, + type ToolDisclosureSummary, +} from "./types"; + +export { buildConservativeClaims } from "./statistics"; + +export interface MarkdownReportOptions { + artifacts?: readonly EvidenceArtifact[]; +} + +/** Render a public report using only schema fields that cannot contain task content. */ +export function renderToolDisclosureMarkdown( + manifest: ToolDisclosureManifest, + summary: ToolDisclosureSummary, + options: MarkdownReportOptions = {}, +): string { + validateInputs(manifest, summary, options.artifacts ?? []); + const claims = buildConservativeClaims(summary); + if (JSON.stringify(claims) !== JSON.stringify(summary.claims)) { + throw new Error("summary claims are stale or were not generated by the claim gate"); + } + + const lines: string[] = [ + "# Progressive Tool-Disclosure Benchmark", + "", + `Schema: \`${TOOL_DISCLOSURE_SCHEMA_VERSION}\` `, + `Benchmark: \`${safeInline(manifest.benchmark_id)}\` `, + `Generated: ${safeInline(summary.generated_at)} `, + `SUT: \`${safeInline(manifest.sut.git_ref)}\` at \`${safeInline(manifest.sut.git_sha)}\` `, + `Harness: \`${safeInline(manifest.harness.git_ref)}\` at \`${safeInline( + manifest.harness.git_sha, + )}\``, + "", + "## Validated claims", + "", + ]; + + if (claims.length === 0) { + lines.push( + "No cross-agent improvement claim cleared all two-campaign gates. See the measured results and gate table below.", + ); + } else { + for (const claim of claims) lines.push(`- ${claim}`); + } + + lines.push( + "", + "These claims cover a deterministic synthetic tool catalog and the recorded system, model, hardware, and task corpus only. Progressive disclosure is a context optimization, not an authorization boundary. The OpenClaw native-catalog results do not establish compaction of mcporter-managed MCP tools.", + "", + "## Test environment", + "", + "| Field | Recorded value |", + "| --- | --- |", + `| GPU | ${cell(`${manifest.environment.gpu_count} × ${manifest.environment.gpu_model}`)} |`, + `| GPU architecture | ${cell(manifest.environment.gpu_architecture)} |`, + `| Driver / CUDA | ${cell( + `${manifest.environment.gpu_driver_version} / ${manifest.environment.cuda_version}`, + )} |`, + `| CPU / RAM | ${cell( + `${manifest.environment.cpu_count} × ${manifest.environment.cpu_model}; ${formatNumber( + manifest.environment.ram_gib, + )} GiB`, + )} |`, + `| OS / architecture | ${cell( + `${manifest.environment.operating_system} / ${manifest.environment.architecture}`, + )} |`, + `| Model | ${cell(`${manifest.inference.model_id} @ ${manifest.inference.model_revision}`)} |`, + `| vLLM image | ${cell( + `${manifest.inference.container_image} @ ${manifest.inference.container_digest}`, + )} |`, + `| vLLM version | ${cell(manifest.inference.vllm_version)} |`, + `| Tool / reasoning parser | ${cell( + `${manifest.inference.tool_call_parser} / ${manifest.inference.reasoning_parser}`, + )} |`, + `| Public vLLM flags | ${cell(manifest.inference.public_vllm_flags.join(" ") || "(none)")} |`, + `| OpenShell | ${cell(manifest.environment.openshell_version)} |`, + `| Agent versions | ${cell( + manifest.protocol.agents + .map((agent) => `${agent}=${manifest.environment.agent_versions[agent]}`) + .join(", "), + )} |`, + `| Power state | ${cell(manifest.environment.power_state)} |`, + "", + "## Exact initial schema visibility", + "", + "| Campaign | Agent | Catalog tools | Direct tokens | Progressive tokens | Reduction | Direct tools | Progressive tools |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ); + + for (const visibility of summary.static_visibility) { + lines.push( + `| ${cell(visibility.campaign_id)} | ${cell(visibility.agent)} | ${visibility.catalog_size} | ` + + `${visibility.direct.tokenizer_tokens} | ${visibility.progressive.tokenizer_tokens} | ` + + `${formatPercent(visibility.reduction.tokenizer_tokens_percent)} | ` + + `${visibility.direct.tool_count} | ${visibility.progressive.tool_count} |`, + ); + } + if (summary.static_visibility.length === 0) + lines.push("| (no measurements) | — | — | — | — | — | — | — |"); + + lines.push( + "", + "Token counts above are exact captures. If a cell has repeated captures, the summary rejects it when byte, token, or tool counts differ.", + "", + "## Paired task results", + "", + "All differences are progressive minus direct. Negative token and latency values favor progressive disclosure; positive success values favor progressive disclosure.", + "", + "| Campaign | Phase | Agent | Catalog | Paired tasks | Success Δ pp (95% CI) | Initial schema token Δ (95% CI) | Total prompt token Δ (95% CI) | First response byte Δ ms (95% CI) | E2E latency Δ ms (95% CI) |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ); + for (const result of summary.comparison_cells) { + const success = result.differences.success_percentage_points; + lines.push( + `| ${cell(result.campaign_id)} | ${cell(result.phase)} | ${cell(result.agent)} | ` + + `${result.catalog_size} | ${success.paired_tasks} | ${formatInterval(success)} | ` + + `${formatInterval(result.differences.initial_tool_schema_tokens)} | ` + + `${formatOptionalInterval(result.differences.total_prompt_tokens)} | ` + + `${formatOptionalInterval(result.differences.time_to_first_response_byte_ms)} | ` + + `${formatOptionalInterval(result.differences.end_to_end_time_ms)} |`, + ); + } + if (summary.comparison_cells.length === 0) { + lines.push("| (no measurements) | — | — | — | — | — | — | — | — | — |"); + } + + const gates = summary.claim_gates; + lines.push( + "", + "## Public claim gates", + "", + `Success noninferiority requires the paired 95% CI lower bound to be at least ${formatNumber( + gates.noninferiority_margin_percentage_points, + )} percentage points. Improvement claims require a paired 95% CI wholly below zero in each of exactly two campaigns for every named agent.`, + "", + "| Campaign | Agent | Success noninferior | Initial schema tokens improved | E2E latency improved | Reasons when blocked |", + "| --- | --- | --- | --- | --- | --- |", + ); + for (const gate of gates.campaign_agent_gates) { + lines.push( + `| ${cell(gate.campaign_id)} | ${cell(gate.agent)} | ${passFail( + gate.success_noninferior, + )} | ${passFail(gate.initial_schema_tokens_improved)} | ${passFail( + gate.end_to_end_latency_improved, + )} | ${cell(gate.reasons.join("; ") || "—")} |`, + ); + } + lines.push( + "", + `- Cross-agent success noninferiority: ${passFail(gates.cross_agent_success_noninferior)}`, + `- Cross-agent initial serialized tool-schema token improvement: ${passFail( + gates.cross_agent_initial_schema_tokens_improved, + )}`, + `- Cross-agent end-to-end latency improvement: ${passFail( + gates.cross_agent_end_to_end_latency_improved, + )}`, + "", + "## Method", + "", + `Task-level progressive/direct pairs were resampled with a deterministic paired bootstrap (${summary.bootstrap_samples.toLocaleString( + "en-US", + )} samples, seed ${summary.bootstrap_seed}). Repetitions were first averaged within each task, so tasks—not individual attempts—are the statistical unit. Setup failures are excluded; timeouts, incorrect calls, model errors, and context overflows are scored failures.`, + "", + "First-response-byte timing is reported only for streaming inference requests. It is transport TTFB, not a claim about the first decoded content token.", + "", + "The static catalog is synthetic. Exact tool names, arguments, nonces, prompts, response bodies, endpoint URLs, credentials, hostnames, usernames, and local paths are not included in this report or its numeric summary.", + ); + + const artifacts = options.artifacts ?? []; + if (artifacts.length > 0) { + lines.push( + "", + "## Evidence artifacts", + "", + "| Artifact | Kind | Bytes | SHA-256 |", + "| --- | --- | ---: | --- |", + ); + for (const artifact of artifacts) { + lines.push( + `| ${cell(artifact.file_name)} | ${cell(artifact.kind)} | ${artifact.byte_length} | \`${artifact.sha256}\` |`, + ); + } + } + + return `${lines.join("\n")}\n`; +} + +function validateInputs( + manifest: ToolDisclosureManifest, + summary: ToolDisclosureSummary, + artifacts: readonly EvidenceArtifact[], +): void { + if ( + manifest.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION || + summary.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION + ) { + throw new Error("manifest and summary must use the tool-disclosure benchmark schema"); + } + if (manifest.benchmark_id !== summary.benchmark_id) { + throw new Error("manifest and summary benchmark IDs do not match"); + } + for (const artifact of artifacts) { + if (artifact.file_name.includes("/") || artifact.file_name.includes("\\")) { + throw new Error(`artifact ${artifact.artifact_id} must use a leaf file name`); + } + if (!/^[a-f0-9]{64}$/.test(artifact.sha256)) { + throw new Error(`artifact ${artifact.artifact_id} has an invalid SHA-256`); + } + } +} + +function formatInterval(interval: ConfidenceInterval): string { + return `${formatNumber(interval.estimate)} [${formatNumber(interval.lower_95)}, ${formatNumber( + interval.upper_95, + )}]`; +} + +function formatOptionalInterval(interval: ConfidenceInterval | undefined): string { + return interval ? formatInterval(interval) : "—"; +} + +function formatPercent(value: number): string { + return `${formatNumber(value)}%`; +} + +function formatNumber(value: number): string { + if (Number.isInteger(value)) return String(value); + return value.toFixed(3).replace(/0+$/, "").replace(/\.$/, ""); +} + +function passFail(value: boolean): string { + return value ? "PASS" : "BLOCKED"; +} + +function safeInline(value: string): string { + return value + .replace(/[\r\n`]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function cell(value: string): string { + return safeInline(value).replaceAll("|", "\\|") || "—"; +} diff --git a/scripts/bench/tool-disclosure/run.ts b/scripts/bench/tool-disclosure/run.ts new file mode 100644 index 0000000000..e6f92f5168 --- /dev/null +++ b/scripts/bench/tool-disclosure/run.ts @@ -0,0 +1,714 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + artifactPath, + ensureCampaignDirectory, + readJsonLines, + scanArtifactsForForbiddenValues, + sha256File, + writeChecksumManifest, + writeJsonArtifact, + writeTextArtifact, +} from "./artifacts"; +import { + assertValidSyntheticCatalog, + DEFAULT_SYNTHETIC_CATALOG_SEED, + generateCatalogPrefix, + generateSyntheticCatalog, + type SyntheticCatalog, +} from "./catalog"; +import { + type AttemptJournalEntry, + type CampaignAttestation, + executeCampaign, + type LiveCampaignConfiguration, + type SanitizedRunEvidence, +} from "./execute"; +import { writeOpenClawFixture } from "./openclaw-fixture"; +import { renderToolDisclosureMarkdown } from "./report"; +import { + buildToolDisclosureSchedule, + STATIC_CATALOG_SIZES, + TOOL_DISCLOSURE_AGENTS, + TOOL_DISCLOSURE_MODES, +} from "./schedule"; +import { buildToolDisclosureSummary } from "./statistics"; +import { + assertValidSyntheticTaskSet, + generatePrimaryTaskSet, + generateStressTaskSet, + type SyntheticTaskSet, +} from "./tasks"; +import { + DEFAULT_BOOTSTRAP_SAMPLES, + DEFAULT_BOOTSTRAP_SEED, + DEFAULT_NONINFERIORITY_MARGIN_PP, + type EvidenceArtifact, + type EvidenceBundle, + TOOL_DISCLOSURE_SCHEMA_VERSION, + type ToolDisclosureManifest, + type ToolDisclosureRun, +} from "./types"; +import { validateCompleteEvidence, validateFrozenManifest } from "./validation"; + +interface ParsedArguments { + command: "prepare" | "execute" | "summarize" | "help"; + outputDir?: string; + sutRef: string; + catalogSeed: string; + executionSeed: number; + sandboxBase?: string; + resume: boolean; + allowDirty: boolean; + forbiddenValues: string[]; + configPath?: string; +} + +interface FixtureManifestEntry { + catalog_size: number; + files: Array<{ path: string; byte_length: number; sha256: string }>; +} + +const PREPARED_FILES = [ + "catalog.json", + "primary-tasks.json", + "stress-tasks.json", + "schedule.json", + "manifest.template.json", + "execute-config.template.json", + "fixtures.json", +] as const; + +function usage(): string { + return `Progressive tool-disclosure benchmark\n\n\ +Usage:\n\ + npm run bench:tool-disclosure -- prepare --output-dir [options]\n\ + npm run bench:tool-disclosure -- execute --output-dir --config \n\ + npm run bench:tool-disclosure -- summarize --output-dir [options]\n\n\ +Prepare options:\n\ + --sut-ref SUT revision to resolve (default: HEAD)\n\ + --catalog-seed Deterministic catalog seed\n\ + --seed Schedule seed\n\ + --sandbox-base Required immutable base image with @sha256 digest\n\ + --resume Reuse an existing output directory\n\n\ + --allow-dirty Development only; resulting evidence cannot be summarized\n\n\ +Summarize inputs:\n\ + Manifest, attestations, attempt journal, raw events, and runs in the output directory\n\ + --forbid-value Fail if a public artifact contains this value (repeatable)\n`; +} + +function requiredValue(args: readonly string[], index: number, flag: string): string { + const value = args[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`${flag} requires a value`); + return value; +} + +function parseArguments(args: readonly string[]): ParsedArguments { + const first = args[0]; + const command = + first === "prepare" || first === "execute" || first === "summarize" ? first : "help"; + if (first && command === "help" && first !== "help" && first !== "--help" && first !== "-h") { + throw new Error(`unknown command: ${first}`); + } + const parsed: ParsedArguments = { + command, + sutRef: "HEAD", + catalogSeed: DEFAULT_SYNTHETIC_CATALOG_SEED, + executionSeed: 0x4e_56_44_41, + resume: false, + allowDirty: false, + forbiddenValues: [], + }; + for (let index = 1; index < args.length; index += 1) { + const flag = args[index]; + if (flag === "--resume") { + parsed.resume = true; + continue; + } + if (flag === "--allow-dirty") { + parsed.allowDirty = true; + continue; + } + if (flag === "--help" || flag === "-h") { + parsed.command = "help"; + continue; + } + const value = requiredValue(args, index, flag); + index += 1; + if (flag === "--output-dir") parsed.outputDir = value; + else if (flag === "--sut-ref") parsed.sutRef = value; + else if (flag === "--catalog-seed") parsed.catalogSeed = value; + else if (flag === "--sandbox-base") parsed.sandboxBase = value; + else if (flag === "--forbid-value") parsed.forbiddenValues.push(value); + else if (flag === "--config") parsed.configPath = value; + else if (flag === "--seed") { + parsed.executionSeed = Number(value); + if (!Number.isSafeInteger(parsed.executionSeed)) throw new Error("--seed must be an integer"); + } else throw new Error(`unknown option: ${flag}`); + } + if (parsed.command !== "help" && !parsed.outputDir) throw new Error("--output-dir is required"); + if (parsed.command === "execute" && !parsed.configPath) + throw new Error("execute requires --config"); + return parsed; +} + +function gitOutput(args: readonly string[]): string { + return execFileSync("git", args, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); +} + +function revision(ref: string, worktreeClean: boolean): ToolDisclosureManifest["sut"] { + const sha = gitOutput(["rev-parse", "--verify", `${ref}^{commit}`]); + if (!/^[a-f0-9]{40}$/u.test(sha)) throw new Error(`could not resolve git revision ${ref}`); + return { git_sha: sha, git_ref: ref, worktree_clean: worktreeClean }; +} + +function benchmarkId(sha: string, catalogHash: string): string { + return `tool-disclosure-${sha.slice(0, 12)}-${catalogHash.slice(0, 12)}`; +} + +function manifestTemplate(options: { + sutRef: string; + catalogHash: string; + executionSeed: number; + worktreeClean: boolean; + tasks: readonly { + id: string; + kind: ToolDisclosureManifest["protocol"]["tasks"][number]["kind"]; + }[]; +}): ToolDisclosureManifest { + const sut = revision(options.sutRef, options.worktreeClean); + const harness = revision("HEAD", options.worktreeClean); + return { + schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, + benchmark_id: benchmarkId(sut.git_sha, options.catalogHash), + created_at: new Date().toISOString(), + sut, + harness, + campaigns: [ + { + campaign_id: "campaign-1", + ordinal: 1, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + { + campaign_id: "campaign-2", + ordinal: 2, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + ], + protocol: { + agents: TOOL_DISCLOSURE_AGENTS, + modes: TOOL_DISCLOSURE_MODES, + catalog_sizes: STATIC_CATALOG_SIZES, + primary_catalog_size: 512, + tasks: options.tasks.map((task) => ({ task_id: task.id, kind: task.kind })), + repetitions: { "small-control": 1, primary: 5, "large-stress": 1 }, + execution_seed: options.executionSeed, + bootstrap_samples: DEFAULT_BOOTSTRAP_SAMPLES, + bootstrap_seed: DEFAULT_BOOTSTRAP_SEED, + noninferiority_margin_percentage_points: DEFAULT_NONINFERIORITY_MARGIN_PP, + retry_setup_failures: 1, + }, + environment: { + operating_system: "RECORD_ON_DGX", + architecture: "RECORD_ON_DGX", + cpu_model: "RECORD_ON_DGX", + cpu_count: 0, + ram_gib: 0, + gpu_model: "RECORD_ON_DGX", + gpu_architecture: "RECORD_ON_DGX", + gpu_count: 0, + gpu_driver_version: "RECORD_ON_DGX", + cuda_version: "RECORD_ON_DGX", + power_state: "RECORD_ON_DGX", + openshell_version: "RECORD_ON_DGX", + agent_versions: { + openclaw: "RECORD_ON_DGX", + hermes: "RECORD_ON_DGX", + "langchain-deepagents-code": "RECORD_ON_DGX", + }, + sandbox_image_digests: Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [`${agent}:${mode}:${size}`, "RECORD_ON_DGX"]), + ), + ), + ), + }, + inference: { + api: "chat-completions", + model_id: "RECORD_ON_DGX", + model_revision: "RECORD_ON_DGX", + container_image: "RECORD_ON_DGX", + container_digest: "RECORD_ON_DGX", + vllm_version: "RECORD_ON_DGX", + tool_call_parser: "RECORD_ON_DGX", + reasoning_parser: "RECORD_ON_DGX", + temperature: 0, + concurrency: 1, + prefix_caching_enabled: false, + public_vllm_flags: [], + }, + }; +} + +function prepare(options: ParsedArguments): void { + const repositoryRoot = path.resolve(gitOutput(["rev-parse", "--show-toplevel"])); + const requestedOutput = path.resolve(options.outputDir as string); + if ( + requestedOutput === repositoryRoot || + requestedOutput.startsWith(`${repositoryRoot}${path.sep}`) + ) { + throw new Error("benchmark output directory must be outside the Git worktree"); + } + const worktreeClean = gitOutput(["status", "--porcelain=v1", "--untracked-files=normal"]) === ""; + if (!worktreeClean && !options.allowDirty) { + throw new Error( + "benchmark preparation requires a clean worktree; commit or stash changes first", + ); + } + if ( + !options.sandboxBase || + (!options.allowDirty && !/@sha256:[a-f0-9]{64}$/u.test(options.sandboxBase)) + ) { + throw new Error("--sandbox-base is required and must end in an immutable @sha256 digest"); + } + const outputDir = ensureCampaignDirectory(requestedOutput, options.resume); + const catalog = generateSyntheticCatalog({ seed: options.catalogSeed }); + const primary = generatePrimaryTaskSet(catalog); + const stress = generateStressTaskSet(catalog); + const schedule = buildToolDisclosureSchedule({ + primaryTaskIds: primary.tasks.map((task) => task.id), + stressTaskIds: stress.tasks.map((task) => task.id), + seed: options.executionSeed, + }); + const manifest = manifestTemplate({ + sutRef: options.sutRef, + catalogHash: catalog.tools_sha256, + executionSeed: options.executionSeed, + worktreeClean, + tasks: [...primary.tasks, ...stress.tasks], + }); + writeJsonArtifact(outputDir, "catalog.json", catalog); + writeJsonArtifact(outputDir, "primary-tasks.json", primary); + writeJsonArtifact(outputDir, "stress-tasks.json", stress); + writeJsonArtifact(outputDir, "schedule.json", schedule); + writeJsonArtifact(outputDir, "manifest.template.json", manifest); + writeJsonArtifact(outputDir, "execute-config.template.json", { + campaign: 1, + upstream_vllm_url: "http://127.0.0.1:8000", + telemetry_url: "http://127.0.0.1:8000", + tokenizer_model: "REPLACE_WITH_MODEL_ID", + recorder_port: 18_080, + managed_inference_base_url: "http://127.0.0.1:18080", + vllm_container_name: "replace-vllm-container-name", + vllm_container_id: "replace-vllm-container-id", + sandbox_names: Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + `replace-${agent}-${mode}-${size}`, + ]), + ), + ), + ), + sandbox_instance_ids: Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + `replace-live-instance-id-${agent}-${mode}-${size}`, + ]), + ), + ), + ), + sandbox_container_names: Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + `replace-live-container-${agent}-${mode}-${size}`, + ]), + ), + ), + ), + timeout_ms: 600_000, + } satisfies LiveCampaignConfiguration); + + for (const size of STATIC_CATALOG_SIZES) { + const fixtureDir = path.join(outputDir, `openclaw-${size}`); + if (fs.existsSync(fixtureDir) && fs.readdirSync(fixtureDir).length > 0) { + const existing = JSON.parse( + fs.readFileSync(path.join(fixtureDir, "plugin", "catalog.json"), "utf8"), + ) as { tools_sha256?: string }; + const expected = generateCatalogPrefix(catalog, size); + if (existing.tools_sha256 !== expected.tools_sha256) { + throw new Error(`existing OpenClaw fixture ${size} does not match the prepared catalog`); + } + continue; + } + writeOpenClawFixture({ + outputDir: fixtureDir, + catalog: generateCatalogPrefix(catalog, size), + sandboxBase: options.sandboxBase, + }); + } + writeJsonArtifact( + outputDir, + "fixtures.json", + STATIC_CATALOG_SIZES.map((size) => { + const fixtureRoot = path.join(outputDir, `openclaw-${size}`); + const files = [ + ".dockerignore", + "Dockerfile", + "plugin/catalog.json", + "plugin/index.js", + "plugin/openclaw.plugin.json", + "plugin/package.json", + ]; + return { + catalog_size: size, + files: files.map((name) => { + const file = path.join(fixtureRoot, name); + return { + path: `openclaw-${size}/${name}`, + byte_length: fs.statSync(file).size, + sha256: sha256File(file), + }; + }), + }; + }), + ); + writeChecksumManifest(outputDir, PREPARED_FILES); + process.stdout.write( + `Prepared ${schedule.length} runs in ${outputDir}. Copy manifest.template.json to manifest.json and replace every RECORD_ON_DGX value before execution.\n`, + ); +} + +function readJson(file: string): T { + try { + return JSON.parse(fs.readFileSync(file, "utf8")) as T; + } catch (error) { + throw new Error(`could not read ${path.basename(file)}: ${String(error)}`); + } +} + +function evidenceArtifact( + outputDir: string, + fileName: string, + kind: EvidenceArtifact["kind"], +): EvidenceArtifact { + const file = artifactPath(outputDir, fileName); + return { + artifact_id: `${kind}-${fileName}`, + kind, + file_name: fileName, + media_type: fileName.endsWith(".jsonl") + ? "application/x-ndjson" + : fileName.endsWith(".md") + ? "text/markdown" + : "application/json", + byte_length: fs.statSync(file).size, + sha256: sha256File(file), + }; +} + +function assertManifestCompleted(manifest: ToolDisclosureManifest): void { + const serialized = JSON.stringify(manifest); + if (serialized.includes("RECORD_ON_DGX")) { + throw new Error("manifest.json still contains RECORD_ON_DGX placeholders"); + } +} + +function assertPreparedIdentity( + manifest: ToolDisclosureManifest, + catalog: SyntheticCatalog, + primaryTasks: SyntheticTaskSet, + stressTasks: SyntheticTaskSet, +): void { + if (manifest.benchmark_id !== benchmarkId(manifest.sut.git_sha, catalog.tools_sha256)) { + throw new Error("manifest benchmark ID does not match the frozen SUT and catalog"); + } + const expectedTasks = [...primaryTasks.tasks, ...stressTasks.tasks].map((task) => ({ + task_id: task.id, + kind: task.kind, + })); + if (JSON.stringify(manifest.protocol.tasks) !== JSON.stringify(expectedTasks)) { + throw new Error("manifest tasks do not match the prepared task artifacts"); + } +} + +export function validateCampaignAttestations( + manifest: ToolDisclosureManifest, + attestations: readonly CampaignAttestation[], +): void { + if ( + attestations.length !== 2 || + JSON.stringify(attestations.map((item) => item.campaign_id).sort()) !== + JSON.stringify(["campaign-1", "campaign-2"]) + ) { + throw new Error("evidence requires exactly two campaign attestations"); + } + if (new Set(attestations.map((item) => item.vllm_process_start_time_seconds)).size !== 2) { + throw new Error("campaign attestations reused a vLLM process"); + } + if ( + new Set(attestations.map((item) => item.inference_container_id_sha256)).size !== 2 || + new Set(attestations.map((item) => item.inference_config_sha256)).size !== 1 || + attestations.some( + (item) => + !/^[a-f0-9]{64}$/u.test(item.inference_container_id_sha256) || + !/^[a-f0-9]{64}$/u.test(item.inference_config_sha256) || + item.inference_image_digest !== manifest.inference.container_digest, + ) + ) { + throw new Error("campaigns require distinct vLLM containers with identical frozen config"); + } + const allInstanceIds = attestations.flatMap((item) => + item.sandbox_cells.map((cell) => cell.instance_id_sha256), + ); + const expectedCells = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => `${agent}:${mode}:${size}`), + ), + ).sort(); + if ( + attestations.some( + (item) => + !Number.isFinite(item.vllm_process_start_time_seconds) || + item.vllm_process_start_time_seconds <= 0 || + JSON.stringify(item.sandbox_cells.map((cell) => cell.cell).sort()) !== + JSON.stringify(expectedCells) || + item.sandbox_cells.some( + (cell) => + !/^[a-f0-9]{64}$/u.test(cell.instance_id_sha256) || + !/^[a-f0-9]{64}$/u.test(cell.status_sha256), + ), + ) || + new Set(allInstanceIds).size !== 60 + ) { + throw new Error("campaign attestations reused or omitted sandbox instances"); + } + for (const attestation of attestations) { + for (const cell of attestation.sandbox_cells) { + if (manifest.environment.sandbox_image_digests?.[cell.cell] !== cell.image_digest) { + throw new Error(`campaign attestation image mismatch for ${cell.cell}`); + } + } + } +} + +function validateOpenClawFixtures( + outputDir: string, + catalog: SyntheticCatalog, + fixtureManifest: readonly FixtureManifestEntry[], +): void { + if ( + !sameNumberList( + fixtureManifest.map((entry) => entry.catalog_size), + STATIC_CATALOG_SIZES, + ) + ) { + throw new Error("fixtures.json does not contain the frozen catalog sizes"); + } + for (const entry of fixtureManifest) { + const root = path.join(outputDir, `openclaw-${entry.catalog_size}`); + const dockerfile = fs.readFileSync(path.join(root, "Dockerfile"), "utf8"); + const sandboxBase = dockerfile.match(/^ARG SANDBOX_BASE=(\S+)$/mu)?.[1]; + if (!sandboxBase) throw new Error(`OpenClaw ${entry.catalog_size} fixture has no base image`); + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fixture-verify-")); + const expectedRoot = path.join(temporaryRoot, `fixture-${entry.catalog_size}`); + try { + const generated = writeOpenClawFixture({ + outputDir: expectedRoot, + catalog: generateCatalogPrefix(catalog, entry.catalog_size), + sandboxBase, + }); + if (entry.files.length !== generated.files.length) { + throw new Error(`OpenClaw ${entry.catalog_size} fixture file count changed`); + } + for (const name of generated.files) { + const relative = `openclaw-${entry.catalog_size}/${name}`; + const recorded = entry.files.find((file) => file.path === relative); + const actual = path.join(root, name); + const expected = path.join(expectedRoot, name); + if ( + !recorded || + recorded.byte_length !== fs.statSync(actual).size || + recorded.sha256 !== sha256File(actual) || + recorded.sha256 !== sha256File(expected) + ) { + throw new Error(`OpenClaw fixture integrity check failed for ${relative}`); + } + } + } finally { + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } + } +} + +function sameNumberList(left: readonly number[], right: readonly number[]): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +function summarize(options: ParsedArguments): void { + const outputDir = path.resolve(options.outputDir as string); + const manifest = readJson(artifactPath(outputDir, "manifest.json")); + assertManifestCompleted(manifest); + validateFrozenManifest(manifest); + const catalog = readJson(artifactPath(outputDir, "catalog.json")); + const primaryTasks = readJson(artifactPath(outputDir, "primary-tasks.json")); + const stressTasks = readJson(artifactPath(outputDir, "stress-tasks.json")); + const fixtures = readJson(artifactPath(outputDir, "fixtures.json")); + const schedule = readJson>( + artifactPath(outputDir, "schedule.json"), + ); + assertValidSyntheticCatalog(catalog); + assertValidSyntheticTaskSet(primaryTasks, catalog); + assertValidSyntheticTaskSet(stressTasks, catalog); + assertPreparedIdentity(manifest, catalog, primaryTasks, stressTasks); + validateOpenClawFixtures(outputDir, catalog, fixtures); + const runs = readJsonLines(artifactPath(outputDir, "runs.jsonl")); + const rawEvidence = readJsonLines( + artifactPath(outputDir, "raw-events.jsonl"), + ); + const journal = readJsonLines( + artifactPath(outputDir, "attempt-journal.jsonl"), + ); + if ( + JSON.stringify(journal.map((entry) => entry.raw)) !== JSON.stringify(rawEvidence) || + JSON.stringify(journal.map((entry) => entry.run)) !== JSON.stringify(runs) + ) { + throw new Error("materialized evidence differs from the atomic attempt journal"); + } + const attestations = readJson( + artifactPath(outputDir, "campaign-attestations.json"), + ); + validateCampaignAttestations(manifest, attestations); + validateCompleteEvidence({ + manifest, + runs, + schedule, + primaryTasks, + stressTasks, + rawEvidence, + catalog, + }); + const summary = buildToolDisclosureSummary(manifest, runs); + writeJsonArtifact(outputDir, "summary.json", summary); + const reportInputs = [ + evidenceArtifact(outputDir, "catalog.json", "catalog"), + evidenceArtifact(outputDir, "primary-tasks.json", "tasks"), + evidenceArtifact(outputDir, "stress-tasks.json", "tasks"), + evidenceArtifact(outputDir, "schedule.json", "schedule"), + evidenceArtifact(outputDir, "fixtures.json", "fixture-manifest"), + evidenceArtifact(outputDir, "manifest.json", "manifest"), + evidenceArtifact(outputDir, "raw-events.jsonl", "raw-events"), + evidenceArtifact(outputDir, "attempt-journal.jsonl", "attempt-journal"), + evidenceArtifact(outputDir, "campaign-attestations.json", "attestation"), + evidenceArtifact(outputDir, "runs.jsonl", "runs"), + evidenceArtifact(outputDir, "summary.json", "summary"), + ]; + writeTextArtifact( + outputDir, + "report.md", + renderToolDisclosureMarkdown(manifest, summary, { artifacts: reportInputs }), + ); + const artifacts = [...reportInputs, evidenceArtifact(outputDir, "report.md", "report")]; + const bundle: EvidenceBundle = { + schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, + benchmark_id: manifest.benchmark_id, + generated_at: new Date().toISOString(), + artifacts, + }; + writeJsonArtifact(outputDir, "evidence.json", bundle); + const publicNames = [ + "catalog.json", + "primary-tasks.json", + "stress-tasks.json", + "schedule.json", + "fixtures.json", + "manifest.json", + "raw-events.jsonl", + "attempt-journal.jsonl", + "campaign-attestations.json", + "runs.jsonl", + "summary.json", + "report.md", + "evidence.json", + ]; + scanArtifactsForForbiddenValues(outputDir, publicNames, options.forbiddenValues); + writeChecksumManifest(outputDir, publicNames); + process.stdout.write( + `Summarized ${runs.length} runs. ${summary.claims.length} claim(s) cleared all gates.\n`, + ); +} + +async function execute(options: ParsedArguments): Promise { + const outputDir = path.resolve(options.outputDir as string); + const manifest = readJson(artifactPath(outputDir, "manifest.json")); + assertManifestCompleted(manifest); + validateFrozenManifest(manifest); + if ( + gitOutput(["rev-parse", "HEAD"]) !== manifest.harness.git_sha || + gitOutput(["status", "--porcelain=v1", "--untracked-files=normal"]) !== "" + ) { + throw new Error("execute requires the clean harness HEAD recorded in manifest.json"); + } + const catalog = readJson(artifactPath(outputDir, "catalog.json")); + const primaryTasks = readJson(artifactPath(outputDir, "primary-tasks.json")); + const stressTasks = readJson(artifactPath(outputDir, "stress-tasks.json")); + const schedule = readJson>( + artifactPath(outputDir, "schedule.json"), + ); + assertValidSyntheticCatalog(catalog); + assertValidSyntheticTaskSet(primaryTasks, catalog); + assertValidSyntheticTaskSet(stressTasks, catalog); + assertPreparedIdentity(manifest, catalog, primaryTasks, stressTasks); + const expectedSchedule = buildToolDisclosureSchedule({ + primaryTaskIds: primaryTasks.tasks.map((task) => task.id), + stressTaskIds: stressTasks.tasks.map((task) => task.id), + seed: manifest.protocol.execution_seed, + }); + if (JSON.stringify(schedule) !== JSON.stringify(expectedSchedule)) { + throw new Error("execute refused a modified schedule.json"); + } + await executeCampaign({ + outputDir, + config: readJson(path.resolve(options.configPath as string)), + manifest, + catalog, + primaryTasks, + stressTasks, + schedule, + }); +} + +export async function main(args = process.argv.slice(2)): Promise { + const options = parseArguments(args); + if (options.command === "help") { + process.stdout.write(usage()); + return; + } + if (options.command === "prepare") prepare(options); + else if (options.command === "execute") await execute(options); + else summarize(options); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + void main().catch((error: unknown) => { + process.stderr.write( + `tool-disclosure benchmark: ${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exitCode = 1; + }); +} diff --git a/scripts/bench/tool-disclosure/schedule.ts b/scripts/bench/tool-disclosure/schedule.ts new file mode 100644 index 0000000000..0743b72611 --- /dev/null +++ b/scripts/bench/tool-disclosure/schedule.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const TOOL_DISCLOSURE_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +export const TOOL_DISCLOSURE_MODES = ["progressive", "direct"] as const; +export const STATIC_CATALOG_SIZES = [16, 64, 256, 512, 2_209] as const; + +export type ToolDisclosureAgent = (typeof TOOL_DISCLOSURE_AGENTS)[number]; +export type ToolDisclosureMode = (typeof TOOL_DISCLOSURE_MODES)[number]; +export type ToolDisclosurePhase = + | "static-visibility" + | "small-control" + | "primary" + | "large-stress"; + +export interface ScheduledToolDisclosureRun { + run_id: string; + campaign: 1 | 2; + phase: ToolDisclosurePhase; + agent: ToolDisclosureAgent; + mode: ToolDisclosureMode; + catalog_size: number; + repetition: number; + task_id?: string; + pair_id: string; + scored: boolean; +} + +export interface ToolDisclosureScheduleOptions { + primaryTaskIds: readonly string[]; + stressTaskIds: readonly string[]; + seed: number; + campaigns?: readonly (1 | 2)[]; +} + +interface PairBlock { + sortKey: number; + runs: [ScheduledToolDisclosureRun, ScheduledToolDisclosureRun]; +} + +function xorshift32(value: number): number { + let state = value | 0; + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return state >>> 0; +} + +function stableHash(text: string, seed: number): number { + let state = seed >>> 0; + for (const char of text) { + state = xorshift32(state ^ (char.codePointAt(0) ?? 0)); + } + return state; +} + +function runId( + campaign: number, + phase: ToolDisclosurePhase, + agent: ToolDisclosureAgent, + mode: ToolDisclosureMode, + catalogSize: number, + taskId: string | undefined, + repetition: number, +): string { + return [ + `c${campaign}`, + phase, + agent, + mode, + `n${catalogSize}`, + taskId ?? "capture", + `r${repetition}`, + ].join("--"); +} + +function makePair( + campaign: 1 | 2, + phase: ToolDisclosurePhase, + agent: ToolDisclosureAgent, + catalogSize: number, + taskId: string | undefined, + repetition: number, + seed: number, +): PairBlock { + const pairId = [ + `c${campaign}`, + phase, + agent, + `n${catalogSize}`, + taskId ?? "capture", + `r${repetition}`, + ].join("--"); + const modes: [ToolDisclosureMode, ToolDisclosureMode] = + stableHash(pairId, seed) % 2 === 0 ? ["progressive", "direct"] : ["direct", "progressive"]; + const scored = phase !== "static-visibility"; + return { + sortKey: stableHash(`sort:${pairId}`, seed), + runs: modes.map((mode) => ({ + run_id: runId(campaign, phase, agent, mode, catalogSize, taskId, repetition), + campaign, + phase, + agent, + mode, + catalog_size: catalogSize, + repetition, + ...(taskId ? { task_id: taskId } : {}), + pair_id: pairId, + scored, + })) as [ScheduledToolDisclosureRun, ScheduledToolDisclosureRun], + }; +} + +function assertUniqueNonEmpty(values: readonly string[], label: string): void { + if (values.length === 0) throw new Error(`${label} must not be empty`); + const normalized = values.map((value) => value.trim()); + if (normalized.some((value) => !value)) throw new Error(`${label} contains an empty id`); + if (new Set(normalized).size !== normalized.length) + throw new Error(`${label} contains duplicate ids`); +} + +export function buildToolDisclosureSchedule( + options: ToolDisclosureScheduleOptions, +): ScheduledToolDisclosureRun[] { + assertUniqueNonEmpty(options.primaryTaskIds, "primaryTaskIds"); + assertUniqueNonEmpty(options.stressTaskIds, "stressTaskIds"); + if (options.primaryTaskIds.length !== 24) { + throw new Error( + `primaryTaskIds must contain exactly 24 tasks, got ${options.primaryTaskIds.length}`, + ); + } + if (options.stressTaskIds.length !== 8) { + throw new Error( + `stressTaskIds must contain exactly 8 tasks, got ${options.stressTaskIds.length}`, + ); + } + if (!Number.isSafeInteger(options.seed)) throw new Error("seed must be a safe integer"); + + const campaigns = options.campaigns ?? [1, 2]; + if (campaigns.length === 0 || campaigns.some((value) => value !== 1 && value !== 2)) { + throw new Error("campaigns must contain campaign 1, campaign 2, or both"); + } + + const scheduled: ScheduledToolDisclosureRun[] = []; + for (const campaign of campaigns) { + const staticBlocks: PairBlock[] = []; + const liveBlocks: PairBlock[] = []; + for (const agent of TOOL_DISCLOSURE_AGENTS) { + for (const catalogSize of STATIC_CATALOG_SIZES) { + staticBlocks.push( + makePair(campaign, "static-visibility", agent, catalogSize, undefined, 1, options.seed), + ); + } + for (const taskId of options.primaryTaskIds) { + liveBlocks.push(makePair(campaign, "small-control", agent, 64, taskId, 1, options.seed)); + for (let repetition = 1; repetition <= 5; repetition += 1) { + liveBlocks.push( + makePair(campaign, "primary", agent, 512, taskId, repetition, options.seed), + ); + } + } + for (const taskId of options.stressTaskIds) { + liveBlocks.push(makePair(campaign, "large-stress", agent, 2_209, taskId, 1, options.seed)); + } + } + staticBlocks.sort( + (a, b) => a.sortKey - b.sortKey || a.runs[0].pair_id.localeCompare(b.runs[0].pair_id), + ); + liveBlocks.sort( + (a, b) => a.sortKey - b.sortKey || a.runs[0].pair_id.localeCompare(b.runs[0].pair_id), + ); + scheduled.push(...staticBlocks.flatMap((block) => block.runs)); + scheduled.push(...liveBlocks.flatMap((block) => block.runs)); + } + + const ids = scheduled.map((run) => run.run_id); + if (new Set(ids).size !== ids.length) + throw new Error("generated schedule contains duplicate run ids"); + return scheduled; +} + +export function countScheduledRunsByCampaign( + schedule: readonly ScheduledToolDisclosureRun[], +): Record { + const counts: Record = {}; + for (const run of schedule) { + const key = `campaign-${run.campaign}:${run.phase}`; + counts[key] = (counts[key] ?? 0) + 1; + } + return counts; +} diff --git a/scripts/bench/tool-disclosure/statistics.ts b/scripts/bench/tool-disclosure/statistics.ts new file mode 100644 index 0000000000..0239725e76 --- /dev/null +++ b/scripts/bench/tool-disclosure/statistics.ts @@ -0,0 +1,739 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + type BenchmarkPhase, + type CampaignAgentGate, + type ClaimGateSummary, + type ComparisonCellSummary, + type ConfidenceInterval, + DEFAULT_BOOTSTRAP_SAMPLES, + DEFAULT_BOOTSTRAP_SEED, + DEFAULT_NONINFERIORITY_MARGIN_PP, + type ModeAggregate, + type SchemaReductionSummary, + type SchemaVisibilitySnapshot, + TOOL_DISCLOSURE_SCHEMA_VERSION, + type ToolDisclosureAgent, + type ToolDisclosureManifest, + type ToolDisclosureMode, + type ToolDisclosureRun, + type ToolDisclosureSummary, +} from "./types"; + +export interface PairedObservation { + progressive: number; + direct: number; +} + +export interface PairedBootstrapOptions { + samples?: number; + seed?: number; +} + +export interface ComparisonOptions extends PairedBootstrapOptions {} + +export interface ClaimGateOptions { + agents: readonly ToolDisclosureAgent[]; + campaignIds: readonly string[]; + primaryCatalogSize: number; + noninferiorityMarginPercentagePoints?: number; +} + +interface TaskAggregate { + success: number; + initialSchemaTokens: number; + totalPromptTokens?: number; + timeToFirstResponseByteMs?: number; + endToEndTimeMs?: number; +} + +interface ComparisonGroup { + campaignId: string; + phase: Exclude; + agent: ToolDisclosureAgent; + catalogSize: number; + runs: ToolDisclosureRun[]; +} + +interface StaticGroup { + campaignId: string; + agent: ToolDisclosureAgent; + catalogSize: number; + runs: ToolDisclosureRun[]; +} + +const PHASE_ORDER: Readonly, number>> = { + "small-control": 0, + primary: 1, + "large-stress": 2, +}; + +/** + * Percentile paired bootstrap over task-level observations. Each bootstrap + * draw resamples whole progressive/direct task pairs, never individual runs. + */ +export function pairedBootstrapDifference( + observations: readonly PairedObservation[], + options: PairedBootstrapOptions = {}, +): ConfidenceInterval { + const samples = options.samples ?? DEFAULT_BOOTSTRAP_SAMPLES; + const seed = normalizeSeed(options.seed ?? DEFAULT_BOOTSTRAP_SEED); + if (!Number.isInteger(samples) || samples <= 0) { + throw new Error(`bootstrap samples must be a positive integer, received ${samples}`); + } + if (observations.length === 0) { + throw new Error("paired bootstrap requires at least one task pair"); + } + for (const observation of observations) { + assertFinite(observation.progressive, "progressive observation"); + assertFinite(observation.direct, "direct observation"); + } + + const differences = observations.map(({ progressive, direct }) => progressive - direct); + const estimate = mean(differences); + const random = mulberry32(seed); + const bootstrapEstimates = new Array(samples); + for (let sampleIndex = 0; sampleIndex < samples; sampleIndex += 1) { + let total = 0; + for (let draw = 0; draw < differences.length; draw += 1) { + total += differences[Math.floor(random() * differences.length)]; + } + bootstrapEstimates[sampleIndex] = total / differences.length; + } + bootstrapEstimates.sort((left, right) => left - right); + + return { + estimate: round6(estimate), + lower_95: round6(quantile(bootstrapEstimates, 0.025)), + upper_95: round6(quantile(bootstrapEstimates, 0.975)), + paired_tasks: observations.length, + bootstrap_samples: samples, + bootstrap_seed: seed, + }; +} + +export function buildComparisonCells( + runs: readonly ToolDisclosureRun[], + options: ComparisonOptions = {}, +): ComparisonCellSummary[] { + const samples = options.samples ?? DEFAULT_BOOTSTRAP_SAMPLES; + const seed = normalizeSeed(options.seed ?? DEFAULT_BOOTSTRAP_SEED); + const groups = new Map(); + + for (const run of runs) { + validateRun(run); + if (run.phase === "static-visibility") continue; + const key = [run.campaign_id, run.phase, run.agent, run.catalog_size].join("\u0000"); + const group = groups.get(key) ?? { + campaignId: run.campaign_id, + phase: run.phase, + agent: run.agent, + catalogSize: run.catalog_size, + runs: [], + }; + group.runs.push(run); + groups.set(key, group); + } + + return [...groups.values()] + .sort(compareComparisonGroups) + .map((group) => buildComparisonCell(group, samples, seed)); +} + +/** + * Produces exact visibility reductions. Repeated captures are accepted only + * when every count is identical, preventing a nondeterministic schema snapshot + * from being rounded into a public claim. + */ +export function summarizeStaticVisibility( + runs: readonly ToolDisclosureRun[], +): SchemaReductionSummary[] { + const groups = new Map(); + for (const run of runs) { + validateRun(run); + if (run.phase !== "static-visibility") continue; + const key = [run.campaign_id, run.agent, run.catalog_size].join("\u0000"); + const group = groups.get(key) ?? { + campaignId: run.campaign_id, + agent: run.agent, + catalogSize: run.catalog_size, + runs: [], + }; + group.runs.push(run); + groups.set(key, group); + } + + return [...groups.values()].sort(compareStaticGroups).map((group) => { + const direct = exactVisibilitySnapshot(group, "direct"); + const progressive = exactVisibilitySnapshot(group, "progressive"); + if (direct.tokenizer_tokens <= 0) { + throw new Error( + `direct static visibility must contain tokenizer tokens for ${staticGroupLabel(group)}`, + ); + } + return { + campaign_id: group.campaignId, + agent: group.agent, + catalog_size: group.catalogSize, + direct, + progressive, + reduction: { + tool_count: direct.tool_count - progressive.tool_count, + serialized_bytes: direct.serialized_bytes - progressive.serialized_bytes, + tokenizer_tokens: direct.tokenizer_tokens - progressive.tokenizer_tokens, + tokenizer_tokens_percent: round6( + ((direct.tokenizer_tokens - progressive.tokenizer_tokens) / direct.tokenizer_tokens) * + 100, + ), + }, + }; + }); +} + +export function evaluateClaimGates( + comparisonCells: readonly ComparisonCellSummary[], + staticVisibility: readonly SchemaReductionSummary[], + options: ClaimGateOptions, +): ClaimGateSummary { + const agents = unique(options.agents); + const campaignIds = unique(options.campaignIds); + const margin = options.noninferiorityMarginPercentagePoints ?? DEFAULT_NONINFERIORITY_MARGIN_PP; + assertFinite(margin, "noninferiority margin"); + const campaignCountValid = campaignIds.length === 2; + const gates: CampaignAgentGate[] = []; + + for (const campaignId of campaignIds) { + for (const agent of agents) { + const matches = comparisonCells.filter( + (cell) => + cell.campaign_id === campaignId && + cell.phase === "primary" && + cell.agent === agent && + cell.catalog_size === options.primaryCatalogSize, + ); + const staticMatches = staticVisibility.filter( + (cell) => + cell.campaign_id === campaignId && + cell.agent === agent && + cell.catalog_size === options.primaryCatalogSize, + ); + const reasons: string[] = []; + if (!campaignCountValid) reasons.push("claim gate requires exactly two campaigns"); + if (matches.length !== 1) { + reasons.push(`expected one primary comparison cell, found ${matches.length}`); + } + if (staticMatches.length !== 1) { + reasons.push( + `expected one deterministic static visibility cell, found ${staticMatches.length}`, + ); + } + + const cell = matches[0]; + const staticCell = staticMatches[0]; + const successLower = cell?.differences.success_percentage_points.lower_95; + const schemaUpper = cell?.differences.initial_tool_schema_tokens.upper_95; + const latencyUpper = cell?.differences.end_to_end_time_ms?.upper_95; + const successNoninferior = + campaignCountValid && + matches.length === 1 && + successLower !== undefined && + cell.differences.success_percentage_points.paired_tasks === 24 && + successLower >= margin; + const schemaImproved = + campaignCountValid && + successNoninferior && + matches.length === 1 && + staticMatches.length === 1 && + schemaUpper !== undefined && + cell.differences.initial_tool_schema_tokens.paired_tasks === 24 && + schemaUpper < 0 && + staticCell.reduction.tokenizer_tokens > 0; + const latencyImproved = + campaignCountValid && + successNoninferior && + matches.length === 1 && + latencyUpper !== undefined && + cell.differences.end_to_end_time_ms?.paired_tasks === 24 && + latencyUpper < 0; + + if (cell && !successNoninferior) { + if (cell.differences.success_percentage_points.paired_tasks !== 24) { + reasons.push("success gate requires exactly 24 paired primary tasks"); + } else { + reasons.push( + `success lower CI ${formatNumber(successLower)} pp is below ${formatNumber(margin)} pp`, + ); + } + } + if (cell && staticCell && !schemaImproved) { + reasons.push( + cell.differences.initial_tool_schema_tokens.paired_tasks !== 24 + ? "schema-token gate requires exactly 24 paired primary tasks" + : "initial schema-token improvement is not significant and repeatable", + ); + } + if (cell && !latencyImproved) { + reasons.push( + cell.differences.end_to_end_time_ms?.paired_tasks !== 24 + ? "latency gate requires exactly 24 paired primary tasks" + : "end-to-end latency improvement requires success noninferiority and a repeatable CI below zero", + ); + } + + gates.push({ + campaign_id: campaignId, + agent, + success_noninferior: successNoninferior, + initial_schema_tokens_improved: schemaImproved, + end_to_end_latency_improved: latencyImproved, + success_lower_95_percentage_points: successLower, + schema_tokens_upper_95: schemaUpper, + latency_upper_95_ms: latencyUpper, + reasons, + }); + } + } + + const completeMatrix = + campaignCountValid && agents.length > 0 && gates.length === agents.length * campaignIds.length; + return { + required_agents: agents, + required_campaigns: campaignIds, + primary_catalog_size: options.primaryCatalogSize, + noninferiority_margin_percentage_points: margin, + campaign_agent_gates: gates, + cross_agent_success_noninferior: + completeMatrix && gates.every((gate) => gate.success_noninferior), + cross_agent_initial_schema_tokens_improved: + completeMatrix && gates.every((gate) => gate.initial_schema_tokens_improved), + cross_agent_end_to_end_latency_improved: + completeMatrix && gates.every((gate) => gate.end_to_end_latency_improved), + }; +} + +export function buildToolDisclosureSummary( + manifest: ToolDisclosureManifest, + runs: readonly ToolDisclosureRun[], + options: { generatedAt?: string } = {}, +): ToolDisclosureSummary { + validateEvidenceSet(manifest, runs); + const comparisonCells = buildComparisonCells(runs, { + samples: manifest.protocol.bootstrap_samples, + seed: manifest.protocol.bootstrap_seed, + }); + const staticVisibility = summarizeStaticVisibility(runs); + const claimGates = evaluateClaimGates(comparisonCells, staticVisibility, { + agents: manifest.protocol.agents, + campaignIds: manifest.campaigns.map((campaign) => campaign.campaign_id), + primaryCatalogSize: manifest.protocol.primary_catalog_size, + noninferiorityMarginPercentagePoints: manifest.protocol.noninferiority_margin_percentage_points, + }); + const summaryWithoutClaims: ToolDisclosureSummary = { + schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, + benchmark_id: manifest.benchmark_id, + generated_at: options.generatedAt ?? new Date().toISOString(), + bootstrap_samples: manifest.protocol.bootstrap_samples, + bootstrap_seed: normalizeSeed(manifest.protocol.bootstrap_seed), + comparison_cells: comparisonCells, + static_visibility: staticVisibility, + claim_gates: claimGates, + claims: [], + }; + return { + ...summaryWithoutClaims, + claims: buildConservativeClaims(summaryWithoutClaims), + }; +} + +/** Public claim text is derived only from gated values; callers cannot supply prose. */ +export function buildConservativeClaims(summary: ToolDisclosureSummary): string[] { + const claims: string[] = []; + const gates = summary.claim_gates; + const targetStaticCells = summary.static_visibility.filter( + (cell) => + gates.required_campaigns.includes(cell.campaign_id) && + gates.required_agents.includes(cell.agent) && + cell.catalog_size === gates.primary_catalog_size, + ); + const expectedCellCount = gates.required_campaigns.length * gates.required_agents.length; + const successMatrixPasses = claimGateMatrixPasses(summary, (gate) => + Boolean( + gate.success_noninferior && + gate.success_lower_95_percentage_points !== undefined && + gate.success_lower_95_percentage_points >= gates.noninferiority_margin_percentage_points, + ), + ); + const schemaMatrixPasses = claimGateMatrixPasses(summary, (gate) => + Boolean( + gate.initial_schema_tokens_improved && + gate.schema_tokens_upper_95 !== undefined && + gate.schema_tokens_upper_95 < 0, + ), + ); + const latencyMatrixPasses = claimGateMatrixPasses(summary, (gate) => + Boolean( + gate.end_to_end_latency_improved && + gate.latency_upper_95_ms !== undefined && + gate.latency_upper_95_ms < 0, + ), + ); + + if ( + gates.cross_agent_initial_schema_tokens_improved && + gates.cross_agent_success_noninferior && + successMatrixPasses && + schemaMatrixPasses && + expectedCellCount > 0 && + targetStaticCells.length === expectedCellCount && + targetStaticCells.every((cell) => cell.reduction.tokenizer_tokens > 0) + ) { + const reductions = targetStaticCells.map((cell) => cell.reduction.tokenizer_tokens_percent); + const minimumReduction = Math.min(...reductions); + const exactNinetyNine = reductions.every((reduction) => reduction >= 99); + const amount = exactNinetyNine + ? "at least 99%" + : `at least ${formatConservativePercent(minimumReduction)}`; + claims.push( + `Across ${gates.required_agents.length} agents and two independent campaigns at ` + + `${gates.primary_catalog_size} catalog tools, progressive disclosure reduced initial ` + + `serialized tool-schema tokens ${amount}.`, + ); + // Defensive invariant: the 99% wording is impossible below the measured threshold. + if (minimumReduction < 99 && claims.at(-1)?.includes("99%")) { + throw new Error("claim generator attempted to overstate schema-token reduction"); + } + } + + if (gates.cross_agent_success_noninferior && successMatrixPasses) { + claims.push( + `Across ${gates.required_agents.length} agents and two independent campaigns at ` + + `${gates.primary_catalog_size} catalog tools, progressive disclosure was noninferior ` + + `to direct exposure for task success at the ${formatNumber( + Math.abs(gates.noninferiority_margin_percentage_points), + )}-percentage-point margin.`, + ); + } + + if ( + gates.cross_agent_end_to_end_latency_improved && + gates.cross_agent_success_noninferior && + successMatrixPasses && + latencyMatrixPasses + ) { + claims.push( + `Across ${gates.required_agents.length} agents and two independent campaigns at ` + + `${gates.primary_catalog_size} catalog tools, progressive disclosure reduced ` + + `end-to-end task latency with paired 95% confidence intervals below zero.`, + ); + } + return claims; +} + +function claimGateMatrixPasses( + summary: ToolDisclosureSummary, + predicate: (gate: CampaignAgentGate) => boolean, +): boolean { + const gates = summary.claim_gates; + if ( + gates.required_campaigns.length !== 2 || + unique(gates.required_campaigns).length !== gates.required_campaigns.length || + gates.required_agents.length === 0 || + unique(gates.required_agents).length !== gates.required_agents.length + ) { + return false; + } + for (const campaignId of gates.required_campaigns) { + for (const agent of gates.required_agents) { + const matching = gates.campaign_agent_gates.filter( + (gate) => gate.campaign_id === campaignId && gate.agent === agent, + ); + if (matching.length !== 1 || !predicate(matching[0])) return false; + } + } + return ( + gates.campaign_agent_gates.length === + gates.required_campaigns.length * gates.required_agents.length + ); +} + +function buildComparisonCell( + group: ComparisonGroup, + samples: number, + baseSeed: number, +): ComparisonCellSummary { + const directRuns = group.runs.filter((run) => run.mode === "direct"); + const progressiveRuns = group.runs.filter((run) => run.mode === "progressive"); + if (directRuns.length === 0 || progressiveRuns.length === 0) { + throw new Error(`comparison requires both modes for ${comparisonGroupLabel(group)}`); + } + const directTasks = aggregateTasks(directRuns); + const progressiveTasks = aggregateTasks(progressiveRuns); + const taskIds = [...directTasks.keys()] + .filter((taskId) => progressiveTasks.has(taskId)) + .sort((left, right) => left.localeCompare(right)); + if (taskIds.length === 0) { + throw new Error(`comparison has no scored task pairs for ${comparisonGroupLabel(group)}`); + } + + const metricSeed = (metric: string): number => + deriveSeed( + baseSeed, + [group.campaignId, group.phase, group.agent, group.catalogSize, metric].join("|"), + ); + const difference = ( + metric: keyof TaskAggregate, + seedName: string, + scale = 1, + ): ConfidenceInterval | undefined => { + const pairs: PairedObservation[] = []; + for (const taskId of taskIds) { + const directValue = directTasks.get(taskId)?.[metric]; + const progressiveValue = progressiveTasks.get(taskId)?.[metric]; + if (directValue === undefined || progressiveValue === undefined) continue; + pairs.push({ direct: directValue * scale, progressive: progressiveValue * scale }); + } + if (pairs.length === 0) return undefined; + return pairedBootstrapDifference(pairs, { samples, seed: metricSeed(seedName) }); + }; + const success = difference("success", "success", 100); + const schema = difference("initialSchemaTokens", "initial-schema-tokens"); + if (!success || !schema) { + throw new Error(`comparison is missing required metrics for ${comparisonGroupLabel(group)}`); + } + + return { + campaign_id: group.campaignId, + phase: group.phase, + agent: group.agent, + catalog_size: group.catalogSize, + direct: aggregateMode(directRuns, directTasks), + progressive: aggregateMode(progressiveRuns, progressiveTasks), + differences: { + success_percentage_points: success, + initial_tool_schema_tokens: schema, + total_prompt_tokens: difference("totalPromptTokens", "total-prompt-tokens"), + time_to_first_response_byte_ms: difference( + "timeToFirstResponseByteMs", + "time-to-first-response-byte", + ), + end_to_end_time_ms: difference("endToEndTimeMs", "end-to-end-time"), + }, + }; +} + +function aggregateTasks(runs: readonly ToolDisclosureRun[]): Map { + const byTask = new Map(); + for (const run of runs) { + if (!run.scored) continue; + const taskRuns = byTask.get(run.task_id) ?? []; + taskRuns.push(run); + byTask.set(run.task_id, taskRuns); + } + return new Map( + [...byTask.entries()].map(([taskId, taskRuns]) => [ + taskId, + { + success: mean(taskRuns.map((run) => Number(run.correctness.task_success))), + initialSchemaTokens: mean( + taskRuns.map((run) => run.measurements.initial_tool_schema.tokenizer_tokens), + ), + totalPromptTokens: optionalMean( + taskRuns.map((run) => run.measurements.total_prompt_tokens), + ), + timeToFirstResponseByteMs: optionalMean( + taskRuns.map((run) => run.measurements.time_to_first_response_byte_ms), + ), + endToEndTimeMs: optionalMean(taskRuns.map((run) => run.measurements.end_to_end_time_ms)), + }, + ]), + ); +} + +function aggregateMode( + runs: readonly ToolDisclosureRun[], + tasks: ReadonlyMap, +): ModeAggregate { + const taskValues = [...tasks.values()]; + return { + runs: runs.length, + scored_runs: runs.filter((run) => run.scored).length, + tasks: tasks.size, + success_rate_percent: round6(mean(taskValues.map((task) => task.success)) * 100), + mean_initial_tool_schema_tokens: round6( + mean(taskValues.map((task) => task.initialSchemaTokens)), + ), + mean_total_prompt_tokens: optionalMean(taskValues.map((task) => task.totalPromptTokens)), + mean_time_to_first_response_byte_ms: optionalMean( + taskValues.map((task) => task.timeToFirstResponseByteMs), + ), + mean_end_to_end_time_ms: optionalMean(taskValues.map((task) => task.endToEndTimeMs)), + }; +} + +function exactVisibilitySnapshot( + group: StaticGroup, + mode: ToolDisclosureMode, +): SchemaVisibilitySnapshot { + const modeRuns = group.runs.filter((run) => run.mode === mode && run.outcome === "success"); + if (modeRuns.length === 0) { + throw new Error(`static visibility is missing ${mode} mode for ${staticGroupLabel(group)}`); + } + const signatures = new Set( + modeRuns.map((run) => { + const value = run.measurements.initial_tool_schema; + return `${value.tool_count}:${value.serialized_bytes}:${value.tokenizer_tokens}`; + }), + ); + if (signatures.size !== 1) { + throw new Error( + `static visibility is nondeterministic for ${mode} mode in ${staticGroupLabel(group)}`, + ); + } + const measurement = modeRuns[0].measurements.initial_tool_schema; + return { ...measurement, samples: modeRuns.length }; +} + +function validateEvidenceSet( + manifest: ToolDisclosureManifest, + runs: readonly ToolDisclosureRun[], +): void { + if (manifest.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION) { + throw new Error(`unsupported manifest schema: ${manifest.schema_version}`); + } + if (manifest.campaigns.length !== 2) { + throw new Error("public claim protocol requires exactly two campaigns"); + } + for (const run of runs) { + if (run.schema_version !== manifest.schema_version) { + throw new Error(`run ${run.run_id} uses a different schema version`); + } + if (run.benchmark_id !== manifest.benchmark_id) { + throw new Error(`run ${run.run_id} belongs to a different benchmark`); + } + } +} + +function validateRun(run: ToolDisclosureRun): void { + if (run.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION) { + throw new Error(`run ${run.run_id} has unsupported schema ${run.schema_version}`); + } + const shouldBeScored = run.phase !== "static-visibility" && run.outcome !== "setup-error"; + if (run.phase !== "static-visibility" && !run.task_kind) { + throw new Error(`run ${run.run_id} is missing its task kind`); + } + if (run.scored !== shouldBeScored) { + throw new Error(`run ${run.run_id} has inconsistent scored/outcome fields`); + } + if ( + run.phase !== "static-visibility" && + run.correctness.task_success !== (run.outcome === "success") + ) { + throw new Error(`run ${run.run_id} has inconsistent success/outcome fields`); + } + for (const [name, value] of Object.entries(run.measurements.initial_tool_schema)) { + assertNonnegativeInteger(value, `${run.run_id} initial_tool_schema.${name}`); + } +} + +function compareComparisonGroups(left: ComparisonGroup, right: ComparisonGroup): number { + return ( + left.campaignId.localeCompare(right.campaignId) || + PHASE_ORDER[left.phase] - PHASE_ORDER[right.phase] || + left.agent.localeCompare(right.agent) || + left.catalogSize - right.catalogSize + ); +} + +function compareStaticGroups(left: StaticGroup, right: StaticGroup): number { + return ( + left.campaignId.localeCompare(right.campaignId) || + left.agent.localeCompare(right.agent) || + left.catalogSize - right.catalogSize + ); +} + +function comparisonGroupLabel(group: ComparisonGroup): string { + return `${group.campaignId}/${group.phase}/${group.agent}/${group.catalogSize}`; +} + +function staticGroupLabel(group: StaticGroup): string { + return `${group.campaignId}/${group.agent}/${group.catalogSize}`; +} + +function optionalMean(values: readonly (number | undefined)[]): number | undefined { + const present = values.filter((value): value is number => value !== undefined); + if (present.length === 0) return undefined; + present.forEach((value) => assertFinite(value, "metric value")); + return round6(mean(present)); +} + +function mean(values: readonly number[]): number { + if (values.length === 0) throw new Error("mean requires at least one value"); + return values.reduce((total, value) => total + value, 0) / values.length; +} + +function quantile(sortedValues: readonly number[], probability: number): number { + const position = (sortedValues.length - 1) * probability; + const lowerIndex = Math.floor(position); + const upperIndex = Math.ceil(position); + const weight = position - lowerIndex; + return sortedValues[lowerIndex] * (1 - weight) + sortedValues[upperIndex] * weight; +} + +function deriveSeed(baseSeed: number, label: string): number { + let hash = normalizeSeed(baseSeed) ^ 0x81_1c_9d_c5; + for (let index = 0; index < label.length; index += 1) { + hash ^= label.charCodeAt(index); + hash = Math.imul(hash, 0x01_00_01_93); + } + return normalizeSeed(hash); +} + +function normalizeSeed(seed: number): number { + if (!Number.isInteger(seed)) + throw new Error(`bootstrap seed must be an integer, received ${seed}`); + return seed >>> 0; +} + +function mulberry32(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d_2b_79_f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296; + }; +} + +function unique(values: readonly T[]): T[] { + return [...new Set(values)]; +} + +function round6(value: number): number { + return Number(value.toFixed(6)); +} + +function formatNumber(value: number | undefined): string { + if (value === undefined) return "unavailable"; + return Number.isInteger(value) + ? String(value) + : value.toFixed(2).replace(/0+$/, "").replace(/\.$/, ""); +} + +function formatConservativePercent(value: number): string { + // One decimal avoids rounding a sub-99 result into the public "99%" headline. + const floored = Math.floor(value * 10) / 10; + return `${formatNumber(floored)}%`; +} + +function assertFinite(value: number, label: string): void { + if (!Number.isFinite(value)) throw new Error(`${label} must be finite`); +} + +function assertNonnegativeInteger(value: number, label: string): void { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${label} must be a nonnegative integer`); + } +} diff --git a/scripts/bench/tool-disclosure/subprocess.ts b/scripts/bench/tool-disclosure/subprocess.ts new file mode 100644 index 0000000000..61044e62fc --- /dev/null +++ b/scripts/bench/tool-disclosure/subprocess.ts @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn } from "node:child_process"; +import { performance } from "node:perf_hooks"; + +export interface SubprocessResult { + exit_code: number | null; + signal: NodeJS.Signals | null; + elapsed_ms: number; + stdout: string; + stderr: string; + timed_out: boolean; + output_truncated: boolean; +} + +function scrub(text: string, redactions: readonly string[]): string { + let result = text; + for (const value of redactions) { + if (value) result = result.replaceAll(value, ""); + } + return result; +} + +export async function runBoundedCommand(options: { + command: string; + args: readonly string[]; + cwd?: string; + env?: NodeJS.ProcessEnv; + timeoutMs: number; + maxOutputBytes?: number; + redactions?: readonly string[]; +}): Promise { + if (!options.command || options.command.includes("\0")) + throw new Error("invalid benchmark command"); + if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1) { + throw new Error("benchmark command timeout must be a positive integer"); + } + const maxOutputBytes = options.maxOutputBytes ?? 2 * 1024 * 1024; + const child = spawn(options.command, [...options.args], { + cwd: options.cwd, + env: options.env, + detached: process.platform !== "win32", + stdio: ["ignore", "pipe", "pipe"], + }); + const startedAt = performance.now(); + let stdout: Buffer = Buffer.alloc(0); + let stderr: Buffer = Buffer.alloc(0); + let truncated = false; + const append = ( + current: Buffer, + chunk: Buffer, + ): Buffer => { + if (current.length >= maxOutputBytes) { + truncated = true; + return current; + } + const remaining = maxOutputBytes - current.length; + if (chunk.length > remaining) truncated = true; + return Buffer.concat([current, chunk.subarray(0, remaining)]); + }; + child.stdout.on("data", (chunk: Buffer) => { + stdout = append(stdout, chunk); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr = append(stderr, chunk); + }); + + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + try { + if (child.pid && process.platform !== "win32") process.kill(-child.pid, "SIGTERM"); + else child.kill("SIGTERM"); + } catch { + // Process already exited. + } + const hardStop = setTimeout(() => { + try { + if (child.pid && process.platform !== "win32") process.kill(-child.pid, "SIGKILL"); + else child.kill("SIGKILL"); + } catch { + // Process already exited. + } + }, 5_000); + hardStop.unref(); + }, options.timeoutMs); + + const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal })); + }, + ).finally(() => clearTimeout(timer)); + const redactions = options.redactions ?? []; + return { + exit_code: result.code, + signal: result.signal, + elapsed_ms: Number((performance.now() - startedAt).toFixed(3)), + stdout: scrub(stdout.toString("utf8"), redactions), + stderr: scrub(stderr.toString("utf8"), redactions), + timed_out: timedOut, + output_truncated: truncated, + }; +} diff --git a/scripts/bench/tool-disclosure/tasks.ts b/scripts/bench/tool-disclosure/tasks.ts new file mode 100644 index 0000000000..4a7d86625b --- /dev/null +++ b/scripts/bench/tool-disclosure/tasks.ts @@ -0,0 +1,368 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + buildSyntheticArguments, + canonicalJson, + executeSyntheticTool, + type JsonObject, + type JsonValue, + SYNTHETIC_CATEGORIES, + type SyntheticCatalog, + type SyntheticTool, + sha256Hex, + validateSyntheticArguments, +} from "./catalog"; + +export const TOOL_DISCLOSURE_TASK_SCHEMA_VERSION = "nemoclaw.tool_disclosure.tasks.v1" as const; +export const PRIMARY_TASK_COUNT = 24; +export const PRIMARY_CATALOG_MIN_SIZE = 64; +export const STRESS_TASK_COUNT = 8; +export const STRESS_CATALOG_SIZE = 2_209; + +export type BenchmarkTaskKind = "single-tool" | "ordered-chain" | "near-match" | "no-tool"; +export type BenchmarkTaskSuite = "primary" | "stress"; + +export interface ExpectedToolCall { + tool_name: string; + arguments: JsonObject; + result_nonce: string; +} + +export interface SyntheticBenchmarkTask { + id: string; + suite: BenchmarkTaskSuite; + kind: BenchmarkTaskKind; + prompt: string; + min_catalog_size: number; + expected_calls: ExpectedToolCall[]; + expected_final_includes: string[]; + distractor_tool_names: string[]; +} + +export interface SyntheticTaskSet { + schema_version: typeof TOOL_DISCLOSURE_TASK_SCHEMA_VERSION; + catalog_seed: string; + suite: BenchmarkTaskSuite; + task_count: number; + tasks_sha256: string; + tasks: SyntheticBenchmarkTask[]; +} + +function toolAt(catalog: SyntheticCatalog, index: number): SyntheticTool { + const tool = catalog.tools[index]; + if (!tool) throw new RangeError(`catalog does not contain required tool index ${index}`); + return tool; +} + +function categoryDetails(tool: SyntheticTool): (typeof SYNTHETIC_CATEGORIES)[number] { + const category = SYNTHETIC_CATEGORIES.find((candidate) => candidate.id === tool.category); + if (!category) throw new Error(`unknown synthetic category ${tool.category}`); + return category; +} + +function expectedCall(tool: SyntheticTool, args: JsonObject): ExpectedToolCall { + return { + tool_name: tool.definition.function.name, + arguments: args, + result_nonce: executeSyntheticTool(tool, args).nonce, + }; +} + +function operationText(tool: SyntheticTool): string { + const category = categoryDetails(tool); + return `the ${tool.operation.replaceAll("_", " ")} operation for a synthetic ${category.resource} in ${category.label}`; +} + +function positiveFinalInstruction(callCount: number): string { + return callCount === 1 + ? "Include the returned nonce exactly in the final answer." + : "Include both returned nonces exactly, in call order, in the final answer."; +} + +function buildSingleTask( + suite: BenchmarkTaskSuite, + ordinal: number, + tool: SyntheticTool, + variant: number, +): SyntheticBenchmarkTask { + const args = buildSyntheticArguments(tool, variant); + const call = expectedCall(tool, args); + return { + id: `${suite}-single-${String(ordinal).padStart(2, "0")}`, + suite, + kind: "single-tool", + prompt: [ + `Use the available tools to perform ${operationText(tool)}.`, + `Pass exactly these arguments: ${canonicalJson(args)}.`, + positiveFinalInstruction(1), + ].join(" "), + min_catalog_size: tool.index + 1, + expected_calls: [call], + expected_final_includes: [call.result_nonce], + distractor_tool_names: [], + }; +} + +function buildChainTask( + ordinal: number, + firstTool: SyntheticTool, + secondTool: SyntheticTool, +): SyntheticBenchmarkTask { + const firstArgs = buildSyntheticArguments(firstTool, 200 + ordinal); + const firstCall = expectedCall(firstTool, firstArgs); + const secondArgs = buildSyntheticArguments(secondTool, 300 + ordinal, { + resource_id: firstCall.result_nonce, + }); + const secondCall = expectedCall(secondTool, secondArgs); + const secondFixedArgs = { ...secondArgs }; + delete secondFixedArgs.resource_id; + + return { + id: `primary-chain-${String(ordinal).padStart(2, "0")}`, + suite: "primary", + kind: "ordered-chain", + prompt: [ + `Complete two tool operations in order. First, perform ${operationText(firstTool)}`, + `with exactly these arguments: ${canonicalJson(firstArgs)}.`, + `Then perform ${operationText(secondTool)}. For the second call, set resource_id to the nonce`, + `returned by the first call and pass these other fields exactly: ${canonicalJson(secondFixedArgs)}.`, + positiveFinalInstruction(2), + ].join(" "), + min_catalog_size: Math.max(firstTool.index, secondTool.index) + 1, + expected_calls: [firstCall, secondCall], + expected_final_includes: [firstCall.result_nonce, secondCall.result_nonce], + distractor_tool_names: [], + }; +} + +function buildNearMatchTask( + ordinal: number, + target: SyntheticTool, + distractors: readonly SyntheticTool[], +): SyntheticBenchmarkTask { + const args = buildSyntheticArguments(target, 400 + ordinal); + const call = expectedCall(target, args); + const distractorOperations = distractors + .map((tool) => tool.operation.replaceAll("_", " ")) + .join(" or "); + + return { + id: `primary-near-match-${String(ordinal).padStart(2, "0")}`, + suite: "primary", + kind: "near-match", + prompt: [ + `Use exactly one tool to perform ${operationText(target)}.`, + `Do not perform the nearby ${distractorOperations} operations in the same category.`, + `Pass exactly these arguments: ${canonicalJson(args)}.`, + positiveFinalInstruction(1), + ].join(" "), + min_catalog_size: Math.max(target.index, ...distractors.map((tool) => tool.index)) + 1, + expected_calls: [call], + expected_final_includes: [call.result_nonce], + distractor_tool_names: distractors.map((tool) => tool.definition.function.name), + }; +} + +const NO_TOOL_CONTROLS = [ + { + phrase: "CONTROL-NO-TOOL-ALPHA", + prompt: "Without using tools, reply with the exact phrase CONTROL-NO-TOOL-ALPHA.", + }, + { + phrase: "CONTROL-NO-TOOL-BRAVO", + prompt: "Do not call a tool. Return only the exact phrase CONTROL-NO-TOOL-BRAVO.", + }, + { + phrase: "CONTROL-NO-TOOL-CHARLIE", + prompt: "This is a no-tool control: answer with exactly CONTROL-NO-TOOL-CHARLIE.", + }, + { + phrase: "CONTROL-NO-TOOL-DELTA", + prompt: "Use no external capability and respond with exactly CONTROL-NO-TOOL-DELTA.", + }, +] as const; + +function buildNoToolTasks(): SyntheticBenchmarkTask[] { + return NO_TOOL_CONTROLS.map((control, index) => ({ + id: `primary-no-tool-${String(index + 1).padStart(2, "0")}`, + suite: "primary", + kind: "no-tool", + prompt: control.prompt, + min_catalog_size: 1, + expected_calls: [], + expected_final_includes: [control.phrase], + distractor_tool_names: [], + })); +} + +export function buildPrimaryTasks(catalog: SyntheticCatalog): SyntheticBenchmarkTask[] { + if (catalog.size < PRIMARY_CATALOG_MIN_SIZE) { + throw new RangeError( + `primary tasks require at least ${PRIMARY_CATALOG_MIN_SIZE} tools; catalog has ${catalog.size}`, + ); + } + + const singleTasks = Array.from({ length: 8 }, (_, index) => + buildSingleTask("primary", index + 1, toolAt(catalog, index), 100 + index), + ); + const chainTasks = Array.from({ length: 8 }, (_, index) => + buildChainTask(index + 1, toolAt(catalog, 8 + index * 2), toolAt(catalog, 9 + index * 2)), + ); + const nearMatchTasks = Array.from({ length: 4 }, (_, index) => + buildNearMatchTask(index + 1, toolAt(catalog, 24 + index), [ + toolAt(catalog, index), + toolAt(catalog, 48 + index), + ]), + ); + return [...singleTasks, ...chainTasks, ...nearMatchTasks, ...buildNoToolTasks()]; +} + +const STRESS_TOOL_INDICES = [256, 511, 767, 1_023, 1_279, 1_535, 1_791, 2_208] as const; + +export function buildStressTasks(catalog: SyntheticCatalog): SyntheticBenchmarkTask[] { + if (catalog.size < STRESS_CATALOG_SIZE) { + throw new RangeError( + `stress tasks require ${STRESS_CATALOG_SIZE} tools; catalog has ${catalog.size}`, + ); + } + return STRESS_TOOL_INDICES.map((toolIndex, index) => + buildSingleTask("stress", index + 1, toolAt(catalog, toolIndex), 500 + index), + ); +} + +function buildTaskSet( + catalog: SyntheticCatalog, + suite: BenchmarkTaskSuite, + tasks: SyntheticBenchmarkTask[], +): SyntheticTaskSet { + return { + schema_version: TOOL_DISCLOSURE_TASK_SCHEMA_VERSION, + catalog_seed: catalog.seed, + suite, + task_count: tasks.length, + tasks_sha256: sha256Hex(canonicalJson(tasks as unknown as JsonValue)), + tasks, + }; +} + +export function generatePrimaryTaskSet(catalog: SyntheticCatalog): SyntheticTaskSet { + return buildTaskSet(catalog, "primary", buildPrimaryTasks(catalog)); +} + +export function generateStressTaskSet(catalog: SyntheticCatalog): SyntheticTaskSet { + return buildTaskSet(catalog, "stress", buildStressTasks(catalog)); +} + +function expectedKindCounts( + suite: BenchmarkTaskSuite, +): Readonly> { + return suite === "primary" + ? { "single-tool": 8, "ordered-chain": 8, "near-match": 4, "no-tool": 4 } + : { "single-tool": 8, "ordered-chain": 0, "near-match": 0, "no-tool": 0 }; +} + +export function validateSyntheticTaskSet( + taskSet: SyntheticTaskSet, + catalog: SyntheticCatalog, +): string[] { + const errors: string[] = []; + if (taskSet.schema_version !== TOOL_DISCLOSURE_TASK_SCHEMA_VERSION) { + errors.push(`unsupported task schema version: ${taskSet.schema_version}`); + } + if (taskSet.catalog_seed !== catalog.seed) errors.push("task and catalog seeds do not match"); + if (taskSet.task_count !== taskSet.tasks.length) errors.push("task_count does not match tasks"); + const expectedHash = sha256Hex(canonicalJson(taskSet.tasks as unknown as JsonValue)); + if (taskSet.tasks_sha256 !== expectedHash) errors.push("tasks_sha256 does not match tasks"); + + const ids = new Set(); + const tools = new Map(catalog.tools.map((tool) => [tool.definition.function.name, tool])); + const counts: Record = { + "single-tool": 0, + "ordered-chain": 0, + "near-match": 0, + "no-tool": 0, + }; + + for (const task of taskSet.tasks) { + counts[task.kind] += 1; + if (ids.has(task.id)) errors.push(`duplicate task id: ${task.id}`); + ids.add(task.id); + if (task.suite !== taskSet.suite) errors.push(`${task.id} has the wrong suite`); + if (task.prompt.trim().length === 0) errors.push(`${task.id} has an empty prompt`); + if (task.min_catalog_size > catalog.size) errors.push(`${task.id} exceeds catalog size`); + + for (const call of task.expected_calls) { + const tool = tools.get(call.tool_name); + if (!tool) { + errors.push(`${task.id} references missing tool ${call.tool_name}`); + continue; + } + const argumentErrors = validateSyntheticArguments(tool, call.arguments); + if (argumentErrors.length > 0) { + errors.push(`${task.id} has invalid arguments for ${call.tool_name}`); + continue; + } + const result = executeSyntheticTool(tool, call.arguments); + if (result.nonce !== call.result_nonce) + errors.push(`${task.id} has an incorrect result nonce`); + if (tool.index + 1 > task.min_catalog_size) + errors.push(`${task.id} has a low min_catalog_size`); + if (!task.expected_final_includes.includes(call.result_nonce)) { + errors.push(`${task.id} does not require its result nonce in the final answer`); + } + } + + if (task.kind === "single-tool" && task.expected_calls.length !== 1) { + errors.push(`${task.id} must expect exactly one call`); + } + if (task.kind === "ordered-chain") { + if (task.expected_calls.length !== 2) { + errors.push(`${task.id} must expect exactly two calls`); + } else if ( + task.expected_calls[1].arguments.resource_id !== task.expected_calls[0].result_nonce + ) { + errors.push(`${task.id} does not pass the first nonce into the second call`); + } + } + if (task.kind === "near-match") { + if (task.expected_calls.length !== 1 || task.distractor_tool_names.length < 1) { + errors.push(`${task.id} must have one expected call and at least one distractor`); + } + for (const distractor of task.distractor_tool_names) { + if (!tools.has(distractor)) + errors.push(`${task.id} references missing distractor ${distractor}`); + if (task.expected_calls.some((call) => call.tool_name === distractor)) { + errors.push(`${task.id} calls a declared distractor`); + } + } + } + if (task.kind === "no-tool" && task.expected_calls.length !== 0) { + errors.push(`${task.id} must not expect tool calls`); + } + } + + const expectedCounts = expectedKindCounts(taskSet.suite); + for (const kind of Object.keys(counts) as BenchmarkTaskKind[]) { + if (counts[kind] !== expectedCounts[kind]) { + errors.push( + `${taskSet.suite} task set has ${counts[kind]} ${kind} tasks; expected ${expectedCounts[kind]}`, + ); + } + } + const expectedTotal = taskSet.suite === "primary" ? PRIMARY_TASK_COUNT : STRESS_TASK_COUNT; + if (taskSet.tasks.length !== expectedTotal) { + errors.push( + `${taskSet.suite} task set has ${taskSet.tasks.length} tasks; expected ${expectedTotal}`, + ); + } + return errors; +} + +export function assertValidSyntheticTaskSet( + taskSet: SyntheticTaskSet, + catalog: SyntheticCatalog, +): void { + const errors = validateSyntheticTaskSet(taskSet, catalog); + if (errors.length > 0) throw new Error(`invalid synthetic task set: ${errors.join("; ")}`); +} diff --git a/scripts/bench/tool-disclosure/telemetry.ts b/scripts/bench/tool-disclosure/telemetry.ts new file mode 100644 index 0000000000..6f41b037ea --- /dev/null +++ b/scripts/bench/tool-disclosure/telemetry.ts @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isIP } from "node:net"; + +const METRIC_NAMES = { + prompt: ["vllm:prompt_tokens_total", "vllm_prompt_tokens_total"], + generation: ["vllm:generation_tokens_total", "vllm_generation_tokens_total"], +} as const; + +export interface VllmTokenSnapshot { + prompt_tokens: number; + generation_tokens: number; +} + +export interface VllmTokenDelta extends VllmTokenSnapshot { + available: boolean; +} + +function isLoopback(hostname: string): boolean { + const normalized = hostname.replace(/^\[/u, "").replace(/\]$/u, "").toLowerCase(); + return ( + normalized === "localhost" || + normalized === "::1" || + (isIP(normalized) === 4 && normalized.startsWith("127.")) + ); +} + +export function validatedTelemetryUrl(raw: string, endpoint: string): URL { + let url: URL; + try { + url = new URL(raw); + } catch { + throw new Error("vLLM telemetry URL must be a valid HTTP(S) URL"); + } + if (!new Set(["http:", "https:"]).has(url.protocol)) { + throw new Error("vLLM telemetry URL must use HTTP or HTTPS"); + } + if (url.username || url.password) { + throw new Error("vLLM telemetry URL must not contain credentials"); + } + if (url.protocol === "http:" && !isLoopback(url.hostname)) { + throw new Error("plaintext vLLM telemetry is allowed only on loopback"); + } + url.username = ""; + url.password = ""; + url.search = ""; + url.hash = ""; + url.pathname = endpoint; + return url; +} + +function metricFamilyValue(text: string, names: readonly string[]): number | undefined { + let total = 0; + let found = false; + for (const line of text.split(/\r?\n/u)) { + if (!line || line.startsWith("#")) continue; + const separator = line.lastIndexOf(" "); + if (separator < 1) continue; + const identity = line.slice(0, separator); + const name = identity.replace(/\{.*$/u, ""); + if (!names.includes(name)) continue; + const value = Number(line.slice(separator + 1)); + if (!Number.isFinite(value) || value < 0) continue; + total += value; + found = true; + } + return found ? total : undefined; +} + +export function parseVllmTokenMetrics(text: string): VllmTokenSnapshot | null { + const prompt = metricFamilyValue(text, METRIC_NAMES.prompt); + const generation = metricFamilyValue(text, METRIC_NAMES.generation); + if (prompt === undefined || generation === undefined) return null; + return { prompt_tokens: prompt, generation_tokens: generation }; +} + +export function parseProcessStartTime(text: string): number | null { + return metricFamilyValue(text, ["process_start_time_seconds"]) ?? null; +} + +export async function readVllmProcessStartTime( + baseUrl: string, + fetchImpl: typeof fetch = fetch, + timeoutMs = 5_000, +): Promise { + const response = await fetchImpl(validatedTelemetryUrl(baseUrl, "/metrics"), { + method: "GET", + redirect: "error", + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`vLLM metrics request failed with HTTP ${response.status}`); + } + const started = parseProcessStartTime(await response.text()); + if (started === null) throw new Error("vLLM metrics did not expose process_start_time_seconds"); + return started; +} + +export async function readVllmTokenSnapshot( + baseUrl: string, + fetchImpl: typeof fetch = fetch, + timeoutMs = 5_000, +): Promise { + const response = await fetchImpl(validatedTelemetryUrl(baseUrl, "/metrics"), { + method: "GET", + redirect: "error", + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + await response.body?.cancel(); + return null; + } + return parseVllmTokenMetrics(await response.text()); +} + +export function tokenDelta( + before: VllmTokenSnapshot | null, + after: VllmTokenSnapshot | null, +): VllmTokenDelta { + if (!before || !after) return { prompt_tokens: 0, generation_tokens: 0, available: false }; + const prompt = after.prompt_tokens - before.prompt_tokens; + const generation = after.generation_tokens - before.generation_tokens; + if (prompt < 0 || generation < 0) { + return { prompt_tokens: 0, generation_tokens: 0, available: false }; + } + return { prompt_tokens: prompt, generation_tokens: generation, available: true }; +} + +function tokenCountFromResponse(payload: unknown): number | null { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; + const record = payload as Record; + for (const key of ["count", "token_count", "num_tokens"]) { + const value = record[key]; + if (Number.isSafeInteger(value) && Number(value) >= 0) return Number(value); + } + const tokens = record.tokens; + return Array.isArray(tokens) ? tokens.length : null; +} + +export async function countTokensWithVllm( + baseUrl: string, + model: string, + text: string, + fetchImpl: typeof fetch = fetch, + timeoutMs = 30_000, +): Promise { + if (!model.trim()) throw new Error("tokenizer model must not be empty"); + const response = await fetchImpl(validatedTelemetryUrl(baseUrl, "/tokenize"), { + method: "POST", + redirect: "error", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model, prompt: text }), + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`vLLM tokenizer request failed with HTTP ${response.status}`); + } + const payload = (await response.json()) as unknown; + const count = tokenCountFromResponse(payload); + if (count === null) throw new Error("vLLM tokenizer response did not contain a token count"); + return count; +} diff --git a/scripts/bench/tool-disclosure/types.ts b/scripts/bench/tool-disclosure/types.ts new file mode 100644 index 0000000000..42a769155c --- /dev/null +++ b/scripts/bench/tool-disclosure/types.ts @@ -0,0 +1,281 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Public evidence schema for the progressive tool-disclosure benchmark. + * + * The schema deliberately stores task identifiers, classifications, booleans, + * and numeric measurements rather than prompts, tool arguments, response + * bodies, endpoint URLs, or credentials. Sanitized transcripts are separate, + * checksummed artifacts and never flow through the statistics/report layer. + */ + +export const TOOL_DISCLOSURE_SCHEMA_VERSION = "nemoclaw.tool_disclosure_bench.v1" as const; +export const DEFAULT_BOOTSTRAP_SAMPLES = 10_000; +export const DEFAULT_BOOTSTRAP_SEED = 0x4e_56_44_41; +export const DEFAULT_NONINFERIORITY_MARGIN_PP = -5; + +export type ToolDisclosureSchemaVersion = typeof TOOL_DISCLOSURE_SCHEMA_VERSION; +export type ToolDisclosureAgent = "langchain-deepagents-code" | "hermes" | "openclaw"; +export type ToolDisclosureMode = "direct" | "progressive"; +export type BenchmarkPhase = "static-visibility" | "small-control" | "primary" | "large-stress"; +export type BenchmarkTaskKind = "single-tool" | "ordered-chain" | "near-match" | "no-tool"; +export type BenchmarkRunOutcome = + | "success" + | "incorrect" + | "timeout" + | "model-error" + | "context-overflow" + | "setup-error"; + +export interface RevisionIdentity { + git_sha: string; + git_ref: string; + worktree_clean: boolean; +} + +export interface CampaignDefinition { + campaign_id: string; + ordinal: 1 | 2; + fresh_inference_process: boolean; + fresh_sandboxes: boolean; +} + +/** Public, sanitized machine metadata. There is intentionally no hostname or path field. */ +export interface BenchmarkEnvironment { + operating_system: string; + architecture: string; + cpu_model: string; + cpu_count: number; + ram_gib: number; + gpu_model: string; + gpu_architecture: string; + gpu_count: number; + gpu_driver_version: string; + cuda_version: string; + power_state: string; + openshell_version: string; + agent_versions: Record; + /** Exact OCI image digest used by every agent/mode/catalog sandbox cell. */ + sandbox_image_digests?: Readonly>; +} + +export interface InferenceConfiguration { + api: "chat-completions"; + model_id: string; + model_revision: string; + container_image: string; + container_digest: string; + vllm_version: string; + tool_call_parser: string; + reasoning_parser: string; + temperature: 0; + concurrency: 1; + prefix_caching_enabled: boolean; + /** Public, reviewed flags only. Never put endpoint, token, mount, or host values here. */ + public_vllm_flags: readonly string[]; +} + +export interface BenchmarkProtocol { + agents: readonly ToolDisclosureAgent[]; + modes: readonly ToolDisclosureMode[]; + catalog_sizes: readonly number[]; + primary_catalog_size: number; + tasks: readonly { + task_id: string; + kind: BenchmarkTaskKind; + }[]; + repetitions: Readonly, number>>; + execution_seed: number; + bootstrap_samples: number; + bootstrap_seed: number; + noninferiority_margin_percentage_points: number; + retry_setup_failures: number; +} + +export interface ToolDisclosureManifest { + schema_version: ToolDisclosureSchemaVersion; + benchmark_id: string; + created_at: string; + sut: RevisionIdentity; + harness: RevisionIdentity; + campaigns: readonly CampaignDefinition[]; + protocol: BenchmarkProtocol; + environment: BenchmarkEnvironment; + inference: InferenceConfiguration; +} + +export interface InitialToolSchemaMeasurement { + tool_count: number; + serialized_bytes: number; + tokenizer_tokens: number; +} + +export interface RunCorrectness { + task_success: boolean; + expected_tool_names: boolean; + expected_tool_order: boolean; + expected_arguments: boolean; + expected_call_count: boolean; + nonce_present: boolean; + unnecessary_tool_calls: number; +} + +export interface RunMeasurements { + initial_tool_schema: InitialToolSchemaMeasurement; + total_prompt_tokens?: number; + completion_tokens?: number; + time_to_first_response_byte_ms?: number; + inference_time_ms?: number; + end_to_end_time_ms?: number; + model_calls: number; + discovery_calls: number; +} + +export interface ToolDisclosureRun { + schema_version: ToolDisclosureSchemaVersion; + benchmark_id: string; + campaign_id: string; + run_id: string; + phase: BenchmarkPhase; + agent: ToolDisclosureAgent; + mode: ToolDisclosureMode; + catalog_size: number; + task_id: string; + task_kind?: BenchmarkTaskKind; + repetition: number; + execution_seed: number; + outcome: BenchmarkRunOutcome; + /** Static captures are unscored; for task phases, only setup-error is excluded. */ + scored: boolean; + correctness: RunCorrectness; + measurements: RunMeasurements; +} + +export interface ConfidenceInterval { + estimate: number; + lower_95: number; + upper_95: number; + paired_tasks: number; + bootstrap_samples: number; + bootstrap_seed: number; +} + +export interface ModeAggregate { + runs: number; + scored_runs: number; + tasks: number; + success_rate_percent: number; + mean_initial_tool_schema_tokens: number; + mean_total_prompt_tokens?: number; + mean_time_to_first_response_byte_ms?: number; + mean_end_to_end_time_ms?: number; +} + +export interface ComparisonDifferences { + /** Progressive minus direct, in percentage points. Higher is better. */ + success_percentage_points: ConfidenceInterval; + /** Progressive minus direct. Negative is better for all remaining metrics. */ + initial_tool_schema_tokens: ConfidenceInterval; + total_prompt_tokens?: ConfidenceInterval; + time_to_first_response_byte_ms?: ConfidenceInterval; + end_to_end_time_ms?: ConfidenceInterval; +} + +export interface ComparisonCellSummary { + campaign_id: string; + phase: Exclude; + agent: ToolDisclosureAgent; + catalog_size: number; + direct: ModeAggregate; + progressive: ModeAggregate; + differences: ComparisonDifferences; +} + +export interface SchemaVisibilitySnapshot extends InitialToolSchemaMeasurement { + samples: number; +} + +export interface SchemaReductionSummary { + campaign_id: string; + agent: ToolDisclosureAgent; + catalog_size: number; + direct: SchemaVisibilitySnapshot; + progressive: SchemaVisibilitySnapshot; + reduction: { + tool_count: number; + serialized_bytes: number; + tokenizer_tokens: number; + tokenizer_tokens_percent: number; + }; +} + +export interface CampaignAgentGate { + campaign_id: string; + agent: ToolDisclosureAgent; + success_noninferior: boolean; + initial_schema_tokens_improved: boolean; + end_to_end_latency_improved: boolean; + success_lower_95_percentage_points?: number; + schema_tokens_upper_95?: number; + latency_upper_95_ms?: number; + reasons: readonly string[]; +} + +export interface ClaimGateSummary { + required_agents: readonly ToolDisclosureAgent[]; + required_campaigns: readonly string[]; + primary_catalog_size: number; + noninferiority_margin_percentage_points: number; + campaign_agent_gates: readonly CampaignAgentGate[]; + cross_agent_success_noninferior: boolean; + cross_agent_initial_schema_tokens_improved: boolean; + cross_agent_end_to_end_latency_improved: boolean; +} + +export interface ToolDisclosureSummary { + schema_version: ToolDisclosureSchemaVersion; + benchmark_id: string; + generated_at: string; + bootstrap_samples: number; + bootstrap_seed: number; + comparison_cells: readonly ComparisonCellSummary[]; + static_visibility: readonly SchemaReductionSummary[]; + claim_gates: ClaimGateSummary; + /** Mechanically generated, public-safe claims. Empty means no claim cleared its gate. */ + claims: readonly string[]; +} + +export type EvidenceArtifactKind = + | "catalog" + | "tasks" + | "schedule" + | "fixture-manifest" + | "manifest" + | "runs" + | "summary" + | "report" + | "sanitized-transcript" + | "raw-events" + | "attempt-journal" + | "attestation" + | "checksums"; + +export interface EvidenceArtifact { + artifact_id: string; + kind: EvidenceArtifactKind; + /** A public-safe leaf name, not an absolute path. */ + file_name: string; + media_type: string; + byte_length: number; + sha256: string; + campaign_id?: string; + run_id?: string; +} + +export interface EvidenceBundle { + schema_version: ToolDisclosureSchemaVersion; + benchmark_id: string; + generated_at: string; + artifacts: readonly EvidenceArtifact[]; +} diff --git a/scripts/bench/tool-disclosure/validation.ts b/scripts/bench/tool-disclosure/validation.ts new file mode 100644 index 0000000000..61354e61ed --- /dev/null +++ b/scripts/bench/tool-disclosure/validation.ts @@ -0,0 +1,358 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { assembleToolDisclosureRun } from "./assemble-run"; +import type { SyntheticCatalog } from "./catalog"; +import type { SanitizedRunEvidence } from "./execute"; +import { + buildToolDisclosureSchedule, + type ScheduledToolDisclosureRun, + STATIC_CATALOG_SIZES, + TOOL_DISCLOSURE_AGENTS, + TOOL_DISCLOSURE_MODES, +} from "./schedule"; +import type { SyntheticTaskSet } from "./tasks"; +import type { ToolDisclosureManifest, ToolDisclosureRun } from "./types"; +import { + DEFAULT_BOOTSTRAP_SAMPLES, + DEFAULT_BOOTSTRAP_SEED, + DEFAULT_NONINFERIORITY_MARGIN_PP, +} from "./types"; + +function sameJson(left: unknown, right: unknown): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +export function validateFrozenManifest(manifest: ToolDisclosureManifest): void { + const requiredEnvironmentText = [ + manifest.environment.operating_system, + manifest.environment.architecture, + manifest.environment.cpu_model, + manifest.environment.gpu_model, + manifest.environment.gpu_architecture, + manifest.environment.gpu_driver_version, + manifest.environment.cuda_version, + manifest.environment.power_state, + manifest.environment.openshell_version, + ...TOOL_DISCLOSURE_AGENTS.map((agent) => manifest.environment.agent_versions[agent]), + ]; + const requiredInferenceText = [ + manifest.inference.model_id, + manifest.inference.model_revision, + manifest.inference.container_image, + manifest.inference.vllm_version, + manifest.inference.tool_call_parser, + manifest.inference.reasoning_parser, + ]; + if ( + manifest.schema_version !== "nemoclaw.tool_disclosure_bench.v1" || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/u.test(manifest.benchmark_id) || + !Number.isFinite(Date.parse(manifest.created_at)) || + !/^[a-f0-9]{40}$/u.test(manifest.sut.git_sha) || + !/^[a-f0-9]{40}$/u.test(manifest.harness.git_sha) || + !manifest.sut.git_ref.trim() || + !manifest.harness.git_ref.trim() || + requiredEnvironmentText.some((value) => typeof value !== "string" || !value.trim()) || + requiredInferenceText.some((value) => typeof value !== "string" || !value.trim()) || + !Number.isSafeInteger(manifest.environment.cpu_count) || + manifest.environment.cpu_count <= 0 || + !Number.isFinite(manifest.environment.ram_gib) || + manifest.environment.ram_gib <= 0 || + !Number.isSafeInteger(manifest.environment.gpu_count) || + manifest.environment.gpu_count <= 0 || + !/^sha256:[a-f0-9]{64}$/u.test(manifest.inference.container_digest) || + manifest.inference.api !== "chat-completions" || + manifest.inference.public_vllm_flags.some( + (flag) => typeof flag !== "string" || !flag.trim() || /[\r\n]/u.test(flag), + ) || + new Set(manifest.inference.public_vllm_flags).size !== + manifest.inference.public_vllm_flags.length || + JSON.stringify(manifest).includes("RECORD_ON_DGX") + ) { + throw new Error("manifest is missing immutable claim-grade environment metadata"); + } + if ( + !sameJson(manifest.protocol.agents, TOOL_DISCLOSURE_AGENTS) || + !sameJson(manifest.protocol.modes, TOOL_DISCLOSURE_MODES) || + !sameJson(manifest.protocol.catalog_sizes, STATIC_CATALOG_SIZES) || + manifest.protocol.primary_catalog_size !== 512 || + !sameJson(manifest.protocol.repetitions, { + "small-control": 1, + primary: 5, + "large-stress": 1, + }) || + manifest.protocol.bootstrap_samples !== DEFAULT_BOOTSTRAP_SAMPLES || + manifest.protocol.bootstrap_seed !== DEFAULT_BOOTSTRAP_SEED || + manifest.protocol.noninferiority_margin_percentage_points !== + DEFAULT_NONINFERIORITY_MARGIN_PP || + manifest.protocol.retry_setup_failures !== 1 + ) { + throw new Error("manifest does not match the frozen benchmark protocol"); + } + if ( + manifest.inference.temperature !== 0 || + manifest.inference.concurrency !== 1 || + manifest.inference.prefix_caching_enabled || + !manifest.sut.worktree_clean || + !manifest.harness.worktree_clean + ) { + throw new Error("manifest does not record clean frozen execution controls"); + } + if ( + !sameJson( + manifest.campaigns.map((campaign) => campaign.campaign_id), + ["campaign-1", "campaign-2"], + ) || + !sameJson( + manifest.campaigns.map((campaign) => campaign.ordinal), + [1, 2], + ) || + manifest.campaigns.some( + (campaign) => !campaign.fresh_inference_process || !campaign.fresh_sandboxes, + ) + ) { + throw new Error("manifest must record two ordered fresh campaigns"); + } + const imageKeys = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => `${agent}:${mode}:${size}`), + ), + ).sort(); + const images = manifest.environment.sandbox_image_digests ?? {}; + if ( + !sameJson(Object.keys(images).sort(), imageKeys) || + Object.values(images).some((digest) => !/^sha256:[a-f0-9]{64}$/u.test(digest)) + ) { + throw new Error("manifest must record an immutable image digest for every sandbox cell"); + } +} + +function expectedCampaignId(run: ScheduledToolDisclosureRun): string { + return `campaign-${run.campaign}`; +} + +function assertRunMatchesSchedule( + run: ToolDisclosureRun, + expected: ScheduledToolDisclosureRun, + taskKinds: ReadonlyMap, +): void { + const expectedTaskId = expected.task_id ?? "static-capture"; + const fields = [ + ["campaign_id", run.campaign_id, expectedCampaignId(expected)], + ["phase", run.phase, expected.phase], + ["agent", run.agent, expected.agent], + ["mode", run.mode, expected.mode], + ["catalog_size", run.catalog_size, expected.catalog_size], + ["task_id", run.task_id, expectedTaskId], + ["repetition", run.repetition, expected.repetition], + ] as const; + for (const [field, actual, wanted] of fields) { + if (actual !== wanted) throw new Error(`run ${run.run_id} has off-schedule ${field}`); + } + const expectedKind = expected.task_id ? taskKinds.get(expected.task_id) : undefined; + if (run.task_kind !== expectedKind) { + throw new Error(`run ${run.run_id} has an off-schedule task kind`); + } +} + +function assertTerminalMeasurements(run: ToolDisclosureRun): void { + const measurements = run.measurements; + const required = [ + ["initial schema bytes", measurements.initial_tool_schema.serialized_bytes], + ["initial schema tokens", measurements.initial_tool_schema.tokenizer_tokens], + ["prompt tokens", measurements.total_prompt_tokens], + ["completion tokens", measurements.completion_tokens], + ["inference time", measurements.inference_time_ms], + ["end-to-end time", measurements.end_to_end_time_ms], + ] as const; + for (const [label, value] of required) { + if (value === undefined || !Number.isFinite(value) || value < 0) { + throw new Error(`run ${run.run_id} has invalid or missing ${label}`); + } + } + if ( + measurements.initial_tool_schema.serialized_bytes === 0 || + measurements.initial_tool_schema.tokenizer_tokens === 0 || + !Number.isSafeInteger(measurements.model_calls) || + measurements.model_calls < 1 || + !Number.isSafeInteger(measurements.discovery_calls) || + measurements.discovery_calls < 0 + ) { + throw new Error(`run ${run.run_id} has invalid required count measurements`); + } +} + +/** Validate the complete frozen run matrix before any public summary is emitted. */ +export function validateCompleteEvidence(options: { + manifest: ToolDisclosureManifest; + runs: readonly ToolDisclosureRun[]; + schedule: readonly ScheduledToolDisclosureRun[]; + primaryTasks: SyntheticTaskSet; + stressTasks: SyntheticTaskSet; + rawEvidence: readonly SanitizedRunEvidence[]; + catalog: SyntheticCatalog; +}): void { + const { manifest, runs, schedule, primaryTasks, stressTasks, rawEvidence, catalog } = options; + validateFrozenManifest(manifest); + if (!sameJson(manifest.protocol.agents, TOOL_DISCLOSURE_AGENTS)) { + throw new Error("manifest agents do not match the frozen protocol"); + } + if (!sameJson(manifest.protocol.modes, TOOL_DISCLOSURE_MODES)) { + throw new Error("manifest modes do not match the frozen protocol"); + } + if (!sameJson(manifest.protocol.catalog_sizes, STATIC_CATALOG_SIZES)) { + throw new Error("manifest catalog sizes do not match the frozen protocol"); + } + if ( + manifest.protocol.primary_catalog_size !== 512 || + !sameJson(manifest.protocol.repetitions, { + "small-control": 1, + primary: 5, + "large-stress": 1, + }) + ) { + throw new Error("manifest repetitions do not match the frozen protocol"); + } + if ( + manifest.protocol.bootstrap_samples !== DEFAULT_BOOTSTRAP_SAMPLES || + manifest.protocol.bootstrap_seed !== DEFAULT_BOOTSTRAP_SEED || + manifest.protocol.noninferiority_margin_percentage_points !== + DEFAULT_NONINFERIORITY_MARGIN_PP || + manifest.protocol.retry_setup_failures !== 1 + ) { + throw new Error("manifest statistics or retry settings do not match the frozen protocol"); + } + if ( + manifest.inference.temperature !== 0 || + manifest.inference.concurrency !== 1 || + manifest.inference.prefix_caching_enabled + ) { + throw new Error("manifest inference controls do not match the frozen protocol"); + } + if (!manifest.sut.worktree_clean || !manifest.harness.worktree_clean) { + throw new Error("public benchmark evidence requires a clean SUT and harness worktree"); + } + const imageKeys = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => `${agent}:${mode}:${size}`), + ), + ).sort(); + const recordedImageKeys = Object.keys(manifest.environment.sandbox_image_digests ?? {}).sort(); + if ( + !sameJson(recordedImageKeys, imageKeys) || + recordedImageKeys.some( + (key) => + !/^sha256:[a-f0-9]{64}$/u.test(manifest.environment.sandbox_image_digests?.[key] ?? ""), + ) + ) { + throw new Error("manifest must record an immutable image digest for every sandbox cell"); + } + if ( + !sameJson( + manifest.campaigns.map((campaign) => campaign.campaign_id), + ["campaign-1", "campaign-2"], + ) || + manifest.campaigns.some( + (campaign) => !campaign.fresh_inference_process || !campaign.fresh_sandboxes, + ) + ) { + throw new Error("manifest must record two fresh frozen campaigns"); + } + + const expectedSchedule = buildToolDisclosureSchedule({ + primaryTaskIds: primaryTasks.tasks.map((task) => task.id), + stressTaskIds: stressTasks.tasks.map((task) => task.id), + seed: manifest.protocol.execution_seed, + }); + if (!sameJson(schedule, expectedSchedule)) { + throw new Error("schedule.json does not match the deterministic frozen schedule"); + } + const allTasks = [...primaryTasks.tasks, ...stressTasks.tasks]; + const expectedProtocolTasks = allTasks.map((task) => ({ task_id: task.id, kind: task.kind })); + if (!sameJson(manifest.protocol.tasks, expectedProtocolTasks)) { + throw new Error("manifest tasks do not match the frozen task artifacts"); + } + const taskKinds = new Map(allTasks.map((task) => [task.id, task.kind] as const)); + const tasksById = new Map(allTasks.map((task) => [task.id, task] as const)); + const expectedById = new Map(schedule.map((run) => [run.run_id, run] as const)); + if (expectedById.size !== schedule.length) throw new Error("schedule contains duplicate run IDs"); + const observedById = new Map(); + if (rawEvidence.length !== runs.length) { + throw new Error("raw-events.jsonl and runs.jsonl have different record counts"); + } + let scheduleCursor = 0; + for (const [index, run] of runs.entries()) { + if (schedule[scheduleCursor]?.run_id !== run.run_id) { + throw new Error(`runs.jsonl is out of frozen schedule order at ${run.run_id}`); + } + const expected = expectedById.get(run.run_id); + if (!expected) throw new Error(`runs.jsonl contains unknown run ID ${run.run_id}`); + assertRunMatchesSchedule(run, expected, taskKinds); + const raw = rawEvidence[index]; + if (raw.run_id !== run.run_id) throw new Error(`raw evidence order differs at ${run.run_id}`); + const task = expected.task_id ? tasksById.get(expected.task_id) : undefined; + const rebuilt = assembleToolDisclosureRun({ + manifest, + scheduled: expected, + task, + calls: raw.calls, + recorderEvents: raw.recorder_events, + invocation: { + ...raw.invocation, + final_output: + raw.final_oracles_present && task ? task.expected_final_includes.join(" ") : "", + }, + initialSchemaTokens: raw.initial_schema_tokens, + ...(raw.prompt_tokens === undefined ? {} : { promptTokens: raw.prompt_tokens }), + ...(raw.completion_tokens === undefined ? {} : { completionTokens: raw.completion_tokens }), + ...(raw.failure_outcome ? { failureOutcome: raw.failure_outcome } : {}), + }); + if (!sameJson(rebuilt, run)) throw new Error(`run ${run.run_id} does not match raw evidence`); + if (run.outcome !== "setup-error") { + const visible = raw.recorder_events.find( + (event) => event.model_call_sequence === 1, + )?.tool_names; + if (!visible) throw new Error(`run ${run.run_id} has no initial visible-tool evidence`); + const synthetic = catalog.tools + .slice(0, run.catalog_size) + .map((tool) => tool.definition.function.name); + const exposed = (name: string): boolean => + visible.some( + (candidate) => + candidate === name || candidate.endsWith(`__${name}`) || candidate.endsWith(`_${name}`), + ); + if (run.mode === "direct" && synthetic.some((name) => !exposed(name))) { + throw new Error(`direct run ${run.run_id} did not expose the complete synthetic catalog`); + } + if (run.mode === "progressive" && synthetic.some(exposed)) { + throw new Error( + `progressive run ${run.run_id} exposed a deferred synthetic tool initially`, + ); + } + } + if (run.execution_seed !== manifest.protocol.execution_seed) { + throw new Error(`run ${run.run_id} has the wrong execution seed`); + } + const observed = observedById.get(run.run_id) ?? []; + if (observed.some((candidate) => sameJson(candidate, run))) { + throw new Error(`runs.jsonl contains an exact duplicate for ${run.run_id}`); + } + observed.push(run); + observedById.set(run.run_id, observed); + if (run.outcome !== "setup-error") scheduleCursor += 1; + } + if (scheduleCursor !== schedule.length) throw new Error("runs.jsonl ended before the schedule"); + + for (const expected of schedule) { + const observed = observedById.get(expected.run_id) ?? []; + const setupFailures = observed.filter((run) => run.outcome === "setup-error"); + const terminal = observed.filter((run) => run.outcome !== "setup-error"); + if (terminal.length !== 1) { + throw new Error(`run ${expected.run_id} must have exactly one terminal record`); + } + if (setupFailures.length > manifest.protocol.retry_setup_failures) { + throw new Error(`run ${expected.run_id} exceeds the setup-failure retry allowance`); + } + assertTerminalMeasurements(terminal[0]); + } +} diff --git a/test/bench/tool-disclosure-catalog.test.ts b/test/bench/tool-disclosure-catalog.test.ts new file mode 100644 index 0000000000..ad54218bb5 --- /dev/null +++ b/test/bench/tool-disclosure-catalog.test.ts @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeAll, describe, expect, it } from "vitest"; + +import { + assertValidSyntheticCatalog, + buildSyntheticArguments, + canonicalJson, + DEFAULT_SYNTHETIC_CATALOG_SEED, + executeSyntheticTool, + generateCatalogPrefix, + generateCatalogPrefixes, + generateSyntheticCatalog, + MAX_SYNTHETIC_TOOLS, + SYNTHETIC_CATEGORIES, + type SyntheticCatalog, + SyntheticToolInvocationError, + sha256Hex, + toOpenAIFunctionTools, + validateSyntheticArguments, + validateSyntheticCatalog, +} from "../../scripts/bench/tool-disclosure/catalog"; +import { + assertValidSyntheticTaskSet, + buildPrimaryTasks, + buildStressTasks, + generatePrimaryTaskSet, + generateStressTaskSet, + PRIMARY_CATALOG_MIN_SIZE, + PRIMARY_TASK_COUNT, + STRESS_CATALOG_SIZE, + STRESS_TASK_COUNT, + validateSyntheticTaskSet, +} from "../../scripts/bench/tool-disclosure/tasks"; + +let fullCatalog: SyntheticCatalog; + +beforeAll(() => { + fullCatalog = generateSyntheticCatalog(); +}); + +describe("synthetic tool-disclosure catalog", () => { + it("documents exactly 24 distinct synthetic categories", () => { + expect(SYNTHETIC_CATEGORIES).toHaveLength(24); + expect(new Set(SYNTHETIC_CATEGORIES.map((category) => category.id)).size).toBe(24); + for (const category of SYNTHETIC_CATEGORIES) { + expect(category.label.length).toBeGreaterThan(0); + expect(category.resource.length).toBeGreaterThan(0); + expect(category.purpose.length).toBeGreaterThan(10); + expect(category.operations).toHaveLength(6); + expect(new Set(category.operations).size).toBe(6); + } + }); + + it("generates 2,209 unique, safe OpenAI-style function definitions", () => { + expect(fullCatalog.seed).toBe(DEFAULT_SYNTHETIC_CATALOG_SEED); + expect(fullCatalog.size).toBe(MAX_SYNTHETIC_TOOLS); + expect(fullCatalog.tools).toHaveLength(2_209); + expect(validateSyntheticCatalog(fullCatalog)).toEqual([]); + expect(() => assertValidSyntheticCatalog(fullCatalog)).not.toThrow(); + + const definitions = toOpenAIFunctionTools(fullCatalog); + const names = definitions.map((definition) => definition.function.name); + expect(definitions).toHaveLength(2_209); + expect(new Set(names).size).toBe(2_209); + expect(names.every((name) => /^[A-Za-z0-9_-]{1,64}$/.test(name))).toBe(true); + expect(definitions.every((definition) => definition.type === "function")).toBe(true); + expect( + definitions.every( + (definition) => + definition.function.description.length > 40 && + definition.function.parameters.type === "object" && + definition.function.parameters.additionalProperties === false, + ), + ).toBe(true); + expect(new Set(fullCatalog.tools.map((tool) => tool.category))).toEqual( + new Set(SYNTHETIC_CATEGORIES.map((category) => category.id)), + ); + }); + + it("uses a stable 25/50/25 schema-complexity cycle with a medium-schema median", () => { + const counts = { small: 0, medium: 0, large: 0 }; + for (const tool of fullCatalog.tools) counts[tool.complexity] += 1; + expect(counts).toEqual({ small: 555, medium: 1_104, large: 550 }); + + const samples = [ + fullCatalog.tools.find((tool) => tool.complexity === "small"), + fullCatalog.tools.find((tool) => tool.complexity === "medium"), + fullCatalog.tools.find((tool) => tool.complexity === "large"), + ]; + for (const tool of samples) { + expect(tool).toBeDefined(); + if (!tool) continue; + const args = buildSyntheticArguments(tool, 17); + expect(validateSyntheticArguments(tool, args)).toEqual([]); + } + expect(Object.keys(samples[0]?.definition.function.parameters.properties ?? {})).toHaveLength( + 2, + ); + expect(Object.keys(samples[1]?.definition.function.parameters.properties ?? {})).toHaveLength( + 5, + ); + expect(Object.keys(samples[2]?.definition.function.parameters.properties ?? {})).toHaveLength( + 8, + ); + }); + + it("is deterministic, seed-sensitive, serializable, and prefix-stable", () => { + const direct64 = generateSyntheticCatalog({ size: 64 }); + const prefix64 = generateCatalogPrefix(fullCatalog, 64); + expect(prefix64).toEqual(direct64); + expect(generateSyntheticCatalog({ size: 64 })).toEqual(direct64); + expect(generateSyntheticCatalog({ size: 64, seed: 41 })).not.toEqual(direct64); + + const requested = generateCatalogPrefixes(fullCatalog, [16, 64, 256, 512, 2_209]); + expect(requested.map((catalog) => catalog.size)).toEqual([16, 64, 256, 512, 2_209]); + for (const catalog of requested) expect(validateSyntheticCatalog(catalog)).toEqual([]); + expect(JSON.parse(JSON.stringify(direct64))).toEqual(direct64); + }); + + it("rejects invalid sizes, seeds, and corrupted catalog metadata", () => { + expect(() => generateSyntheticCatalog({ size: 0 })).toThrow(/catalog size/); + expect(() => generateSyntheticCatalog({ size: 2_210 })).toThrow(/catalog size/); + expect(() => generateSyntheticCatalog({ seed: "" })).toThrow(/seed/); + expect(() => generateSyntheticCatalog({ seed: -1 })).toThrow(/seed/); + expect(() => generateCatalogPrefix(generateSyntheticCatalog({ size: 16 }), 17)).toThrow( + /exceeds/, + ); + expect(validateSyntheticCatalog({ ...fullCatalog, size: 2_208 })).toContain( + "catalog size does not match tool count", + ); + expect(validateSyntheticCatalog({ ...fullCatalog, tools_sha256: "0".repeat(64) })).toContain( + "catalog tools_sha256 does not match tools", + ); + }); +}); + +describe("canonical fixture execution", () => { + it("canonicalizes object keys and produces a stable SHA-256 digest", () => { + const first = { z: [3, { b: true, a: null }], a: "value" }; + const second = { a: "value", z: [3, { a: null, b: true }] }; + expect(canonicalJson(first)).toBe(canonicalJson(second)); + expect(canonicalJson(first)).toBe('{"a":"value","z":[3,{"a":null,"b":true}]}'); + expect(sha256Hex(canonicalJson(first))).toMatch(/^[a-f0-9]{64}$/); + expect(() => canonicalJson(Number.NaN)).toThrow(/non-finite/); + }); + + it("returns deterministic argument-bound nonces from every executable fixture", () => { + for (const tool of fullCatalog.tools) { + const args = buildSyntheticArguments(tool, tool.index % 503); + expect(validateSyntheticArguments(tool, args)).toEqual([]); + const result = executeSyntheticTool(tool, args); + expect(result.ok).toBe(true); + expect(result.tool_name).toBe(tool.definition.function.name); + expect(result.nonce).toMatch(/^nonce_[a-f0-9]{20}$/); + } + + const tool = fullCatalog.tools[17]; + const args = buildSyntheticArguments(tool, 8); + const reordered = Object.fromEntries(Object.entries(args).reverse()); + expect(executeSyntheticTool(tool, reordered).nonce).toBe( + executeSyntheticTool(tool, args).nonce, + ); + expect(executeSyntheticTool(tool, { ...args, query: "a changed request" }).nonce).not.toBe( + executeSyntheticTool(tool, args).nonce, + ); + }); + + it("rejects malformed handler arguments before producing a result", () => { + const tool = fullCatalog.tools.find((candidate) => candidate.complexity === "large"); + expect(tool).toBeDefined(); + if (!tool) return; + const valid = buildSyntheticArguments(tool, 3); + expect(validateSyntheticArguments(tool, { ...valid, unexpected: true })).toContain( + "arguments.unexpected is not allowed", + ); + expect(() => executeSyntheticTool(tool, { ...valid, limit: 0 })).toThrow( + SyntheticToolInvocationError, + ); + }); +}); + +describe("tool-disclosure task fixtures", () => { + it("builds the exact 24-task primary composition with valid oracles", () => { + const taskSet = generatePrimaryTaskSet(fullCatalog); + expect(taskSet.task_count).toBe(PRIMARY_TASK_COUNT); + expect(taskSet.tasks).toHaveLength(24); + expect(validateSyntheticTaskSet(taskSet, fullCatalog)).toEqual([]); + expect(() => assertValidSyntheticTaskSet(taskSet, fullCatalog)).not.toThrow(); + + const count = (kind: string) => taskSet.tasks.filter((task) => task.kind === kind).length; + expect(count("single-tool")).toBe(8); + expect(count("ordered-chain")).toBe(8); + expect(count("near-match")).toBe(4); + expect(count("no-tool")).toBe(4); + expect(new Set(taskSet.tasks.map((task) => task.id)).size).toBe(24); + expect(JSON.parse(JSON.stringify(taskSet))).toEqual(taskSet); + }); + + it("makes every chain consume the first result and every distractor genuinely nearby", () => { + const tasks = buildPrimaryTasks(fullCatalog); + const toolsByName = new Map( + fullCatalog.tools.map((tool) => [tool.definition.function.name, tool]), + ); + + for (const task of tasks.filter((candidate) => candidate.kind === "ordered-chain")) { + expect(task.expected_calls).toHaveLength(2); + expect(task.expected_calls[1].arguments.resource_id).toBe( + task.expected_calls[0].result_nonce, + ); + expect(task.expected_final_includes).toEqual( + task.expected_calls.map((call) => call.result_nonce), + ); + } + for (const task of tasks.filter((candidate) => candidate.kind === "near-match")) { + const target = toolsByName.get(task.expected_calls[0].tool_name); + expect(target).toBeDefined(); + expect(task.distractor_tool_names).toHaveLength(2); + for (const distractorName of task.distractor_tool_names) { + const distractor = toolsByName.get(distractorName); + expect(distractor?.category).toBe(target?.category); + expect(distractor?.operation).not.toBe(target?.operation); + expect(task.expected_calls.map((call) => call.tool_name)).not.toContain(distractorName); + } + } + for (const task of tasks.filter((candidate) => candidate.kind === "no-tool")) { + expect(task.expected_calls).toEqual([]); + expect(task.expected_final_includes).toHaveLength(1); + } + }); + + it("builds eight single-tool stress tasks distributed through the 2,209-tool catalog", () => { + const taskSet = generateStressTaskSet(fullCatalog); + expect(taskSet.task_count).toBe(STRESS_TASK_COUNT); + expect(taskSet.tasks).toHaveLength(8); + expect(taskSet.tasks.every((task) => task.kind === "single-tool")).toBe(true); + expect(taskSet.tasks.map((task) => task.expected_calls[0].tool_name)).toEqual( + [256, 511, 767, 1_023, 1_279, 1_535, 1_791, 2_208].map( + (index) => fullCatalog.tools[index].definition.function.name, + ), + ); + expect(validateSyntheticTaskSet(taskSet, fullCatalog)).toEqual([]); + expect(buildStressTasks(fullCatalog)).toEqual(taskSet.tasks); + }); + + it("keeps primary tasks prefix-invariant and rejects undersized task catalogs", () => { + const prefix = generateCatalogPrefix(fullCatalog, PRIMARY_CATALOG_MIN_SIZE); + expect(generatePrimaryTaskSet(prefix).tasks).toEqual(generatePrimaryTaskSet(fullCatalog).tasks); + expect(() => buildPrimaryTasks(generateSyntheticCatalog({ size: 63 }))).toThrow(/at least 64/); + expect(() => buildStressTasks(generateSyntheticCatalog({ size: 512 }))).toThrow(/require 2209/); + + const valid = generatePrimaryTaskSet(prefix); + expect(validateSyntheticTaskSet({ ...valid, tasks_sha256: "f".repeat(64) }, prefix)).toContain( + "tasks_sha256 does not match tasks", + ); + }); + + it("exposes fixed size constants matching the generated task requirements", () => { + expect(PRIMARY_CATALOG_MIN_SIZE).toBe(64); + expect(STRESS_CATALOG_SIZE).toBe(MAX_SYNTHETIC_TOOLS); + }); +}); diff --git a/test/bench/tool-disclosure-drivers.test.ts b/test/bench/tool-disclosure-drivers.test.ts new file mode 100644 index 0000000000..541a048801 --- /dev/null +++ b/test/bench/tool-disclosure-drivers.test.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildAgentDriverCommand, + extractFinalAssistantOutput, + outputContainsOracle, +} from "../../scripts/bench/tool-disclosure/drivers"; + +describe("tool-disclosure agent drivers", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("builds a bounded sandbox command for %s", (agent) => { + const prompt = "Use the weather converter and return BENCH_OK_123"; + const command = buildAgentDriverCommand({ + agent, + sandboxName: "bench-openclaw-progressive-512", + prompt, + sessionId: "bench-session-1", + }); + expect(command.command).toBe("openshell"); + expect(command.args.slice(0, 5)).toEqual([ + "sandbox", + "exec", + "-n", + "bench-openclaw-progressive-512", + "--", + ]); + expect(command.args.join(" ")).not.toContain(prompt); + expect(command.redactions).toContain(prompt); + }); + + it("uses an exact oracle and rejects unsafe identifiers", () => { + expect(outputContainsOracle("result BENCH_OK_123", "BENCH_OK_123")).toBe(true); + expect(outputContainsOracle("result BENCH_OK_123", "BENCH_OK_456")).toBe(false); + expect(() => + buildAgentDriverCommand({ + agent: "openclaw", + sandboxName: "bad name", + prompt: "x", + sessionId: "one", + }), + ).toThrow("sandboxName contains unsupported characters"); + }); + + it("extracts assistant replies without accepting tool-result fields", () => { + expect( + extractFinalAssistantOutput( + "openclaw", + JSON.stringify({ + result: { payloads: [{ text: "FINAL_NONCE" }] }, + tool_result: { text: "TOOL_ONLY_NONCE" }, + }), + ), + ).toBe("FINAL_NONCE"); + expect( + extractFinalAssistantOutput( + "hermes", + JSON.stringify({ + choices: [{ message: { role: "assistant", content: "FINAL_NONCE" } }], + tool_result: "TOOL_ONLY_NONCE", + }), + ), + ).toBe("FINAL_NONCE"); + }); +}); diff --git a/test/bench/tool-disclosure-execute.test.ts b/test/bench/tool-disclosure-execute.test.ts new file mode 100644 index 0000000000..ac3f230fef --- /dev/null +++ b/test/bench/tool-disclosure-execute.test.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + type AttemptJournalEntry, + type CampaignAttestation, + materializeAttemptJournal, + recoverAttemptJournal, +} from "../../scripts/bench/tool-disclosure/execute"; +import { validateCampaignAttestations } from "../../scripts/bench/tool-disclosure/run"; +import { + STATIC_CATALOG_SIZES, + TOOL_DISCLOSURE_AGENTS, + TOOL_DISCLOSURE_MODES, +} from "../../scripts/bench/tool-disclosure/schedule"; +import type { ToolDisclosureManifest } from "../../scripts/bench/tool-disclosure/types"; + +const hex = (value: number): string => value.toString(16).padStart(64, "0"); + +function attestationFixture(): { + manifest: ToolDisclosureManifest; + attestations: CampaignAttestation[]; +} { + const cells = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => `${agent}:${mode}:${size}`), + ), + ); + const sandboxDigest = `sha256:${"a".repeat(64)}`; + const inferenceDigest = `sha256:${"b".repeat(64)}`; + const manifest = { + inference: { container_digest: inferenceDigest }, + environment: { + sandbox_image_digests: Object.fromEntries(cells.map((cell) => [cell, sandboxDigest])), + }, + } as ToolDisclosureManifest; + const attestations = [1, 2].map( + (campaign): CampaignAttestation => ({ + campaign_id: `campaign-${campaign}`, + vllm_process_start_time_seconds: campaign, + inference_container_id_sha256: hex(campaign), + inference_config_sha256: hex(100), + inference_image_digest: inferenceDigest, + sandbox_cells: cells.map((cell, index) => ({ + cell, + instance_id_sha256: hex(campaign * 100 + index), + status_sha256: hex(campaign * 1_000 + index), + image_digest: sandboxDigest, + })), + }), + ); + return { manifest, attestations }; +} + +describe("tool-disclosure attempt journal", () => { + it("recovers one trailing partial append and deterministically materializes evidence", () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); + try { + const entry = { + raw: { run_id: "run-1", recorder_events: [], calls: [] }, + run: { run_id: "run-1", outcome: "setup-error" }, + } as unknown as AttemptJournalEntry; + fs.writeFileSync( + path.join(output, "attempt-journal.jsonl"), + `${JSON.stringify(entry)}\n{"raw":`, + ); + + const recovered = recoverAttemptJournal(output); + expect(recovered).toEqual([entry]); + expect(fs.readFileSync(path.join(output, "attempt-journal.jsonl"), "utf8")).toBe( + `${JSON.stringify(entry)}\n`, + ); + + materializeAttemptJournal(output, recovered); + expect(JSON.parse(fs.readFileSync(path.join(output, "raw-events.jsonl"), "utf8"))).toEqual( + entry.raw, + ); + expect(JSON.parse(fs.readFileSync(path.join(output, "runs.jsonl"), "utf8"))).toEqual( + entry.run, + ); + } finally { + fs.rmSync(output, { recursive: true, force: true }); + } + }); + + it("requires two fresh campaigns with identical frozen inference configuration", () => { + const { manifest, attestations } = attestationFixture(); + expect(() => validateCampaignAttestations(manifest, attestations)).not.toThrow(); + + const reusedContainer = structuredClone(attestations); + reusedContainer[1].inference_container_id_sha256 = + reusedContainer[0].inference_container_id_sha256; + expect(() => validateCampaignAttestations(manifest, reusedContainer)).toThrow( + /distinct vLLM containers/u, + ); + + const changedConfiguration = structuredClone(attestations); + changedConfiguration[1].inference_config_sha256 = hex(101); + expect(() => validateCampaignAttestations(manifest, changedConfiguration)).toThrow( + /identical frozen config/u, + ); + + const reusedSandbox = structuredClone(attestations); + reusedSandbox[1].sandbox_cells[0].instance_id_sha256 = + reusedSandbox[0].sandbox_cells[0].instance_id_sha256; + expect(() => validateCampaignAttestations(manifest, reusedSandbox)).toThrow( + /reused or omitted sandbox instances/u, + ); + }); +}); diff --git a/test/bench/tool-disclosure-fixture.test.ts b/test/bench/tool-disclosure-fixture.test.ts new file mode 100644 index 0000000000..ef1ec6aff3 --- /dev/null +++ b/test/bench/tool-disclosure-fixture.test.ts @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { assembleToolDisclosureRun } from "../../scripts/bench/tool-disclosure/assemble-run"; +import { + canonicalJson, + executeSyntheticTool, + generateSyntheticCatalog, + sha256Hex, +} from "../../scripts/bench/tool-disclosure/catalog"; +import { gradeTaskRun } from "../../scripts/bench/tool-disclosure/grading"; +import { writeOpenClawFixture } from "../../scripts/bench/tool-disclosure/openclaw-fixture"; +import { buildToolDisclosureSchedule } from "../../scripts/bench/tool-disclosure/schedule"; +import { buildPrimaryTasks } from "../../scripts/bench/tool-disclosure/tasks"; +import { + TOOL_DISCLOSURE_SCHEMA_VERSION, + type ToolDisclosureManifest, +} from "../../scripts/bench/tool-disclosure/types"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + delete process.env.NEMOCLAW_BENCH_CALLS_PATH; + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +describe("OpenClaw benchmark fixture", () => { + it("registers and executes the deterministic native tool catalog", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bench-fixture-")); + temporaryDirectories.push(root); + const output = path.join(root, "fixture-16"); + const catalog = generateSyntheticCatalog({ size: 16 }); + writeOpenClawFixture({ outputDir: output, catalog, sandboxBase: "example/base@sha256:1234" }); + + const registered: Array> = []; + const callsPath = path.join(root, "calls.jsonl"); + process.env.NEMOCLAW_BENCH_CALLS_PATH = callsPath; + const moduleUrl = `${pathToFileURL(path.join(output, "plugin", "index.js")).href}?test=1`; + const plugin = (await import(moduleUrl)) as { + default(api: { registerTool(tool: Record): void }): void; + }; + plugin.default({ registerTool: (tool) => registered.push(tool) }); + expect(registered).toHaveLength(16); + + const first = registered[0] as { + execute(id: string, params: Record): Promise<{ details: unknown }>; + }; + const args = buildPrimaryTasks(generateSyntheticCatalog({ size: 64 }))[0].expected_calls[0] + .arguments; + const result = await first.execute("call-1", args); + expect(result.details).toEqual(executeSyntheticTool(catalog.tools[0], args)); + const call = JSON.parse(fs.readFileSync(callsPath, "utf8")) as Record; + expect(call).toMatchObject({ + tool_name: catalog.tools[0].definition.function.name, + arguments_sha256: sha256Hex(canonicalJson(args)), + success: true, + }); + + const dockerfile = fs.readFileSync(path.join(output, "Dockerfile"), "utf8"); + expect(dockerfile).toContain("ARG NEMOCLAW_TOOL_DISCLOSURE=progressive"); + expect(dockerfile).toContain("ENV NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE}"); + const manifest = JSON.parse( + fs.readFileSync(path.join(output, "plugin", "openclaw.plugin.json"), "utf8"), + ) as { contracts: { tools: string[] } }; + expect(manifest.contracts.tools).toHaveLength(16); + }); +}); + +describe("benchmark grading", () => { + it("requires exact ordered calls, argument hashes, nonces, and final oracle", () => { + const catalog = generateSyntheticCatalog({ size: 64 }); + const task = buildPrimaryTasks(catalog).find((candidate) => candidate.kind === "ordered-chain"); + if (!task) throw new Error("chain task fixture missing"); + const calls = task.expected_calls.map((call) => ({ + tool_name: call.tool_name, + arguments_sha256: sha256Hex(canonicalJson(call.arguments)), + result_nonce: call.result_nonce, + success: true, + })); + const passing = gradeTaskRun(task, calls, task.expected_final_includes.join(" ")); + expect(passing.outcome).toBe("success"); + expect( + gradeTaskRun(task, [...calls].reverse(), task.expected_final_includes.join(" ")), + ).toMatchObject({ outcome: "incorrect", correctness: { expected_tool_order: false } }); + }); + + it("passes a no-tool control only when no calls occur and the phrase is present", () => { + const task = buildPrimaryTasks(generateSyntheticCatalog({ size: 64 })).find( + (candidate) => candidate.kind === "no-tool", + ); + if (!task) throw new Error("no-tool fixture missing"); + expect(gradeTaskRun(task, [], task.expected_final_includes[0]).outcome).toBe("success"); + expect( + gradeTaskRun( + task, + [ + { + tool_name: "unexpected", + arguments_sha256: "0".repeat(64), + result_nonce: null, + success: true, + }, + ], + task.expected_final_includes[0], + ).outcome, + ).toBe("incorrect"); + }); + + it("assembles a public run without retaining model output or arguments", () => { + const catalog = generateSyntheticCatalog({ size: 64 }); + const tasks = buildPrimaryTasks(catalog); + const task = tasks[0]; + const scheduled = buildToolDisclosureSchedule({ + primaryTaskIds: tasks.map((item) => item.id), + stressTaskIds: Array.from({ length: 8 }, (_, index) => `stress-${index}`), + seed: 1, + campaigns: [1], + }).find((run) => run.task_id === task.id && run.agent === "openclaw"); + if (!scheduled) throw new Error("scheduled fixture missing"); + const manifest = { + schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, + benchmark_id: "fixture-benchmark", + campaigns: [ + { + campaign_id: "campaign-1", + ordinal: 1, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + ], + protocol: { execution_seed: 1 }, + } as unknown as ToolDisclosureManifest; + const expected = task.expected_calls[0]; + const record = assembleToolDisclosureRun({ + manifest, + scheduled, + task, + calls: [ + { + tool_name: expected.tool_name, + arguments_sha256: sha256Hex(canonicalJson(expected.arguments)), + result_nonce: expected.result_nonce, + success: true, + }, + ], + recorderEvents: [ + { + run_id: scheduled.run_id, + request_sequence: 1, + model_call_sequence: 1, + endpoint: "chat-completions", + method: "POST", + visible_tool_count: 4, + canonical_tools_json_bytes: 321, + tools_sha256: "a".repeat(64), + tool_names: [expected.tool_name], + streaming: true, + status_code: 200, + started_monotonic_ms: 1, + first_byte_monotonic_ms: 2, + ended_monotonic_ms: 3, + duration_ms: 2, + time_to_first_byte_ms: 1, + outcome: "completed", + error_reason: null, + }, + ], + invocation: { + exit_code: 0, + timed_out: false, + elapsed_ms: 10, + final_output: `private output ${expected.result_nonce}`, + }, + initialSchemaTokens: 42, + }); + expect(record.outcome).toBe("success"); + expect(record.measurements.initial_tool_schema).toEqual({ + tool_count: 4, + serialized_bytes: 321, + tokenizer_tokens: 42, + }); + expect(JSON.stringify(record)).not.toContain("private output"); + expect(JSON.stringify(record)).not.toContain(String(expected.arguments.resource_id)); + }); +}); diff --git a/test/bench/tool-disclosure-mcp-server.test.ts b/test/bench/tool-disclosure-mcp-server.test.ts new file mode 100644 index 0000000000..9bf1f7a275 --- /dev/null +++ b/test/bench/tool-disclosure-mcp-server.test.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildSyntheticArguments, + generateSyntheticCatalog, +} from "../../scripts/bench/tool-disclosure/catalog"; +import { + executeArgumentHash, + SyntheticMcpServer, +} from "../../scripts/bench/tool-disclosure/mcp-server"; + +async function rpc( + url: string, + token: string, + method: string, + params: Record = {}, +): Promise { + return await fetch(url, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }), + }); +} + +describe("synthetic benchmark MCP server", () => { + it("lists and executes the generated catalog while retaining content-free call evidence", async () => { + const catalog = generateSyntheticCatalog({ size: 16, seed: "mcp-server-test" }); + const token = "fixture-token-that-must-not-be-recorded"; + const server = new SyntheticMcpServer(catalog, token); + const address = await server.start(); + try { + const unauthorized = await fetch(address.local_url, { method: "POST", body: "{}" }); + expect(unauthorized.status).toBe(401); + + const initialized = await rpc(address.local_url, token, "initialize", { + protocolVersion: "2025-03-26", + }); + expect((await initialized.json()) as object).toMatchObject({ + result: { serverInfo: { name: "nemoclaw-tool-disclosure-benchmark" } }, + }); + + const listed = await rpc(address.local_url, token, "tools/list"); + const listPayload = (await listed.json()) as { result: { tools: unknown[] } }; + expect(listPayload.result.tools).toHaveLength(16); + + const tool = catalog.tools[0]; + const args = buildSyntheticArguments(tool, 42); + server.beginRun("c1-primary-openclaw-progressive"); + const called = await rpc(address.local_url, token, "tools/call", { + name: tool.definition.function.name, + arguments: args, + }); + const callPayload = (await called.json()) as { result: { content: Array<{ text: string }> } }; + expect(callPayload.result.content[0].text).toContain('"nonce"'); + const events = server.endRun(); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + run_id: "c1-primary-openclaw-progressive", + sequence: 1, + tool_name: tool.definition.function.name, + arguments_sha256: executeArgumentHash(args), + success: true, + }); + expect(JSON.stringify(events)).not.toContain(token); + expect(JSON.stringify(events)).not.toContain(JSON.stringify(args)); + } finally { + await server.stop(); + } + }); +}); diff --git a/test/bench/tool-disclosure-quick-tunnel.test.ts b/test/bench/tool-disclosure-quick-tunnel.test.ts new file mode 100644 index 0000000000..5f62140946 --- /dev/null +++ b/test/bench/tool-disclosure-quick-tunnel.test.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildQuickTunnelArgs, + parseQuickTunnelOrigin, +} from "../../scripts/bench/tool-disclosure/quick-tunnel"; + +describe("tool-disclosure quick tunnel", () => { + it("extracts only a bounded trycloudflare origin", () => { + expect( + parseQuickTunnelOrigin("INF Requesting new quick Tunnel https://bench-123.trycloudflare.com"), + ).toBe("https://bench-123.trycloudflare.com"); + expect(parseQuickTunnelOrigin("https://example.com/secret")).toBeNull(); + }); + + it("builds a loopback origin command and rejects invalid ports", () => { + expect(buildQuickTunnelArgs(31337)).toEqual([ + "tunnel", + "--no-autoupdate", + "--protocol", + "http2", + "--url", + "http://127.0.0.1:31337", + "--loglevel", + "info", + ]); + expect(() => buildQuickTunnelArgs(0)).toThrow("between 1 and 65535"); + }); +}); diff --git a/test/bench/tool-disclosure-recorder.test.ts b/test/bench/tool-disclosure-recorder.test.ts new file mode 100644 index 0000000000..0ef2736cf4 --- /dev/null +++ b/test/bench/tool-disclosure-recorder.test.ts @@ -0,0 +1,457 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import http, { type IncomingMessage, type ServerResponse } from "node:http"; +import type { Socket } from "node:net"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + createToolDisclosureRecordingProxy, + type ToolDisclosureRecordingProxy, +} from "../../scripts/bench/tool-disclosure/recorder"; + +interface TestUpstream { + baseUrl: string; + close: () => Promise; +} + +const cleanup: Array<() => Promise> = []; + +afterEach(async () => { + await Promise.allSettled(cleanup.splice(0).map((close) => close())); +}); + +async function startUpstream( + handler: (request: IncomingMessage, response: ServerResponse) => void, +): Promise { + const sockets = new Set(); + const server = http.createServer(handler); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("test upstream did not bind"); + + const close = async (): Promise => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + }; + cleanup.push(close); + return { baseUrl: `http://127.0.0.1:${address.port}`, close }; +} + +async function startProxy( + upstreamBaseUrl: string, + options: { + maxRequestBodyBytes?: number; + requestTimeoutMs?: number; + requiredTemperature?: number; + } = {}, +): Promise<{ proxy: ToolDisclosureRecordingProxy; baseUrl: string }> { + const proxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl, ...options }); + const address = await proxy.start(); + cleanup.push(() => proxy.stop()); + expect(address.host).toBe("127.0.0.1"); + expect(address.base_url).toBe(`http://127.0.0.1:${address.port}`); + return { proxy, baseUrl: address.base_url }; +} + +function collectBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + request.once("error", reject); + }); +} + +describe("tool-disclosure recording proxy", () => { + it("fails closed for non-loopback listeners and credential-bearing upstream URLs", () => { + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + listenHost: "0.0.0.0", + }), + ).toThrow("must be exactly 127.0.0.1"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://user:password@inference.example/v1", + }), + ).toThrow("must not contain credentials"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1?api_key=private-key", + }), + ).toThrow("must not contain a query or fragment"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://inference.example/v1", + }), + ).toThrow("plaintext upstreamBaseUrl is allowed only on loopback"); + }); + + it("accepts bounded public scheduled run IDs and rejects unsafe IDs", () => { + const proxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + }); + const scheduledId = "c1--primary--openclaw--progressive--n512--single-01--r1"; + expect(proxy.beginRun(scheduledId)).toBe(scheduledId); + expect(proxy.endRun()).toEqual([]); + + expect(() => proxy.beginRun("run id with prompt content")).toThrow("public-safe"); + expect(() => proxy.beginRun(`r${"x".repeat(256)}`)).toThrow("1-256"); + }); + + it("forwards requests while retaining only canonical tool metadata", async () => { + const received: Array<{ url: string; authorization: string; body: string }> = []; + const upstream = await startUpstream((request, response) => { + void collectBody(request).then((body) => { + received.push({ + url: request.url ?? "", + authorization: String(request.headers.authorization ?? ""), + body, + }); + response.writeHead(200, { + "content-type": "application/json", + "x-private-upstream-header": "header-secret", + }); + response.end('{"choices":[{"message":{"content":"response-secret"}}]}'); + }); + }); + const { proxy, baseUrl } = await startProxy(`${upstream.baseUrl}/v1`); + const tool = { + type: "function", + function: { + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + name: "weather_lookup", + description: "schema-secret", + }, + }; + const requestBody = JSON.stringify({ + model: "model-secret", + messages: [{ role: "user", content: "prompt-secret" }], + tools: [tool], + stream: false, + }); + + const runId = proxy.beginRun(); + const firstResponse = await fetch(`${baseUrl}/v1/chat/completions?api_key=query-secret`, { + method: "POST", + headers: { + authorization: "Bearer authorization-secret", + "content-type": "application/json", + }, + body: requestBody, + }); + expect(await firstResponse.text()).toContain("response-secret"); + + const reorderedTool = { + function: { + description: "schema-secret", + name: "weather_lookup", + parameters: { + properties: { city: { type: "string" } }, + type: "object", + }, + }, + type: "function", + }; + const secondResponse = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [], tools: [reorderedTool] }), + }); + await secondResponse.arrayBuffer(); + + const events = proxy.endRun(); + expect(events).toHaveLength(2); + expect(events[0]).toMatchObject({ + run_id: runId, + request_sequence: 1, + model_call_sequence: 1, + endpoint: "chat-completions", + method: "POST", + visible_tool_count: 1, + tool_names: ["weather_lookup"], + status_code: 200, + outcome: "completed", + error_reason: null, + }); + expect(events[1].model_call_sequence).toBe(2); + expect(events[1].tools_sha256).toBe(events[0].tools_sha256); + + const canonical = JSON.stringify([ + { + function: { + description: "schema-secret", + name: "weather_lookup", + parameters: { + properties: { city: { type: "string" } }, + type: "object", + }, + }, + type: "function", + }, + ]); + expect(events[0].canonical_tools_json_bytes).toBe(Buffer.byteLength(canonical)); + expect(events[0].tools_sha256).toBe(createHash("sha256").update(canonical).digest("hex")); + + expect(received[0]).toEqual({ + url: "/v1/chat/completions?api_key=query-secret", + authorization: "Bearer authorization-secret", + body: requestBody, + }); + const serializedEvents = JSON.stringify(events); + for (const secret of [ + "prompt-secret", + "model-secret", + "schema-secret", + "authorization-secret", + "query-secret", + "response-secret", + "header-secret", + ]) { + expect(serializedEvents).not.toContain(secret); + } + + const snapshots = proxy.consumeToolSchemaSnapshots(runId); + expect(snapshots).toHaveLength(2); + expect(snapshots[0]).toEqual({ + run_id: runId, + model_call_sequence: 1, + canonical_tools_json: canonical, + }); + expect(proxy.consumeToolSchemaSnapshots(runId)).toEqual([]); + + proxy.resetEvents(); + expect(proxy.getEvents()).toEqual([]); + }); + + it("streams SSE bytes unchanged and records monotonic first-byte timing", async () => { + const chunks = [ + 'data: {"choices":[{"delta":{"content":"A"}}]}\n\n', + 'data: {"choices":[{"delta":{"content":"B"}}]}\n\n', + "data: [DONE]\n\n", + ]; + const upstream = await startUpstream((_request, response) => { + response.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + }); + response.write(chunks[0]); + setTimeout(() => { + response.write(chunks[1]); + response.end(chunks[2]); + }, 15); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/v1/responses`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "private-model", + input: "private-input", + stream: true, + tools: [{ type: "function", name: "calculator", parameters: {} }], + }), + }); + expect(response.headers.get("content-type")).toBe("text/event-stream"); + expect(await response.text()).toBe(chunks.join("")); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + endpoint: "responses", + visible_tool_count: 1, + tool_names: ["calculator"], + status_code: 200, + outcome: "completed", + }); + expect(event.first_byte_monotonic_ms).not.toBeNull(); + expect(event.started_monotonic_ms).toBeLessThanOrEqual(event.first_byte_monotonic_ms ?? 0); + expect(event.first_byte_monotonic_ms ?? Number.POSITIVE_INFINITY).toBeLessThanOrEqual( + event.ended_monotonic_ms, + ); + expect(event.duration_ms).toBeGreaterThanOrEqual(event.time_to_first_byte_ms ?? 0); + }); + + it("forces a missing frozen temperature and rejects a conflicting value", async () => { + const received: string[] = []; + const upstream = await startUpstream((request, response) => { + void collectBody(request).then((body) => { + received.push(body); + response.end("{}"); + }); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { requiredTemperature: 0 }); + proxy.beginRun("temperature-missing"); + await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [], tools: [] }), + }); + proxy.endRun(); + expect(JSON.parse(received[0])).toMatchObject({ temperature: 0 }); + + proxy.beginRun("temperature-conflict"); + const rejected = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [], tools: [], temperature: 0.5 }), + }); + expect(rejected.status).toBe(400); + proxy.endRun(); + expect(received).toHaveLength(1); + }); + + it("passes redirects through without following them or recording their bodies", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.writeHead(307, { + location: "/v1/redirect-target?token=location-secret", + "content-type": "text/plain", + }); + response.end("redirect-response-secret"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + redirect: "manual", + headers: { "content-type": "application/json" }, + body: '{"messages":[{"content":"request-secret"}]}', + }); + expect(response.status).toBe(307); + expect(response.headers.get("location")).toContain("location-secret"); + expect(await response.text()).toBe("redirect-response-secret"); + expect(hits).toBe(1); + + const [event] = proxy.endRun(); + expect(event.status_code).toBe(307); + expect(event.outcome).toBe("completed"); + expect(JSON.stringify(event)).not.toMatch( + /location-secret|redirect-response-secret|request-secret/, + ); + }); + + it("rejects oversized bodies without contacting the upstream", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + maxRequestBodyBytes: 32, + }); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [{ content: "x".repeat(100) }] }), + }); + expect(response.status).toBe(413); + expect(await response.json()).toEqual({ error: "request body too large" }); + expect(hits).toBe(0); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + status_code: 413, + outcome: "request-rejected", + error_reason: "request-body-too-large", + visible_tool_count: 0, + }); + }); + + it("bounds stalled upstream calls and exposes only a redacted error class", async () => { + const upstream = await startUpstream(async (request, response) => { + await collectBody(request); + setTimeout(() => { + if (!response.destroyed) { + response.writeHead(500, { "content-type": "text/plain" }); + response.end("upstream-error-body-secret"); + } + }, 200); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTimeoutMs: 30, + }); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { + authorization: "Bearer timeout-authorization-secret", + "content-type": "application/json", + }, + body: '{"messages":[{"content":"timeout-prompt-secret"}]}', + }); + expect(response.status).toBe(504); + expect(await response.json()).toEqual({ error: "upstream timeout" }); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + status_code: 504, + outcome: "upstream-timeout", + error_reason: "upstream-timeout", + }); + expect(JSON.stringify(event)).not.toMatch( + /timeout-authorization-secret|timeout-prompt-secret|upstream-error-body-secret/, + ); + }); + + it("bounds an SSE response that stalls after its first byte", async () => { + const upstream = await startUpstream((_request, response) => { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.write("data: first\n\n"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTimeoutMs: 30, + }); + + proxy.beginRun("c1--primary--hermes--progressive--n512--single-01--r1"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"stream":true}', + }); + expect(response.status).toBe(200); + await expect(response.text()).rejects.toBeDefined(); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + status_code: 200, + outcome: "upstream-timeout", + error_reason: "upstream-timeout", + }); + expect(event.first_byte_monotonic_ms).not.toBeNull(); + }); + + it("does not forward or record paths outside the OpenAI v1 boundary", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/health?token=health-secret`); + expect(response.status).toBe(404); + expect(hits).toBe(0); + expect(proxy.endRun()).toEqual([]); + }); +}); diff --git a/test/bench/tool-disclosure-report.test.ts b/test/bench/tool-disclosure-report.test.ts new file mode 100644 index 0000000000..406d2af611 --- /dev/null +++ b/test/bench/tool-disclosure-report.test.ts @@ -0,0 +1,413 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { renderToolDisclosureMarkdown } from "../../scripts/bench/tool-disclosure/report"; +import { + buildComparisonCells, + buildConservativeClaims, + buildToolDisclosureSummary, + evaluateClaimGates, + pairedBootstrapDifference, + summarizeStaticVisibility, +} from "../../scripts/bench/tool-disclosure/statistics"; +import { + type BenchmarkPhase, + DEFAULT_BOOTSTRAP_SAMPLES, + TOOL_DISCLOSURE_SCHEMA_VERSION, + type ToolDisclosureAgent, + type ToolDisclosureManifest, + type ToolDisclosureMode, + type ToolDisclosureRun, +} from "../../scripts/bench/tool-disclosure/types"; + +const AGENTS = ["langchain-deepagents-code", "hermes", "openclaw"] as const; +const CAMPAIGNS = ["campaign-1", "campaign-2"] as const; +const TASK_IDS = Array.from({ length: 24 }, (_, index) => `task-${index + 1}`); + +function makeManifest(overrides: Partial = {}): ToolDisclosureManifest { + return { + schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, + benchmark_id: "tool-disclosure-public-proof", + created_at: "2026-07-06T12:00:00.000Z", + sut: { git_sha: "3a05b54e", git_ref: "v0.0.74", worktree_clean: true }, + harness: { git_sha: "1234abcd", git_ref: "benchmark-harness", worktree_clean: true }, + campaigns: [ + { + campaign_id: "campaign-1", + ordinal: 1, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + { + campaign_id: "campaign-2", + ordinal: 2, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + ], + protocol: { + agents: AGENTS, + modes: ["direct", "progressive"], + catalog_sizes: [512], + primary_catalog_size: 512, + tasks: TASK_IDS.map((taskId) => ({ task_id: taskId, kind: "single-tool" as const })), + repetitions: { "small-control": 1, primary: 1, "large-stress": 1 }, + execution_seed: 7, + bootstrap_samples: 500, + bootstrap_seed: 42, + noninferiority_margin_percentage_points: -5, + retry_setup_failures: 2, + }, + environment: { + operating_system: "Ubuntu 24.04", + architecture: "x86_64", + cpu_model: "DGX Station CPU", + cpu_count: 20, + ram_gib: 128, + gpu_model: "NVIDIA DGX Station GPU", + gpu_architecture: "Blackwell", + gpu_count: 1, + gpu_driver_version: "999.1", + cuda_version: "13.0", + power_state: "maximum-performance", + openshell_version: "0.1.0", + agent_versions: { + "langchain-deepagents-code": "1.0.0", + hermes: "1.0.0", + openclaw: "1.0.0", + }, + }, + inference: { + api: "chat-completions", + model_id: "deepseek-ai/DeepSeek-V4-Flash", + model_revision: "revision-1", + container_image: "nvcr.io/nvidia/vllm:26.05.post1-py3", + container_digest: "sha256:container", + vllm_version: "0.1.0", + tool_call_parser: "deepseek_v3", + reasoning_parser: "deepseek_r1", + temperature: 0, + concurrency: 1, + prefix_caching_enabled: true, + public_vllm_flags: ["--enable-prefix-caching"], + }, + ...overrides, + }; +} + +interface RunOptions { + campaign?: string; + agent?: ToolDisclosureAgent; + mode?: ToolDisclosureMode; + phase?: BenchmarkPhase; + taskId?: string; + repetition?: number; + success?: boolean; + scored?: boolean; + schemaTokens?: number; + schemaBytes?: number; + visibleTools?: number; + totalPromptTokens?: number; + latencyMs?: number; +} + +let runSequence = 0; + +function makeRun(options: RunOptions = {}): ToolDisclosureRun { + const success = options.success ?? true; + const schemaTokens = options.schemaTokens ?? 1_000; + const mode = options.mode ?? "direct"; + const phase = options.phase ?? "primary"; + const scored = options.scored ?? phase !== "static-visibility"; + runSequence += 1; + return { + schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, + benchmark_id: "tool-disclosure-public-proof", + campaign_id: options.campaign ?? "campaign-1", + run_id: `run-${runSequence}`, + phase, + agent: options.agent ?? "langchain-deepagents-code", + mode, + catalog_size: 512, + task_id: options.taskId ?? "task-1", + task_kind: "single-tool", + repetition: options.repetition ?? 1, + execution_seed: 7, + outcome: + phase === "static-visibility" + ? "success" + : scored + ? success + ? "success" + : "incorrect" + : "setup-error", + scored, + correctness: { + task_success: scored && success, + expected_tool_names: scored && success, + expected_tool_order: scored && success, + expected_arguments: scored && success, + expected_call_count: scored && success, + nonce_present: scored && success, + unnecessary_tool_calls: 0, + }, + measurements: { + initial_tool_schema: { + tool_count: options.visibleTools ?? (mode === "direct" ? 512 : 3), + serialized_bytes: options.schemaBytes ?? schemaTokens * 4, + tokenizer_tokens: schemaTokens, + }, + total_prompt_tokens: options.totalPromptTokens ?? schemaTokens + 100, + completion_tokens: 20, + time_to_first_response_byte_ms: (options.latencyMs ?? 200) / 2, + inference_time_ms: options.latencyMs ?? 200, + end_to_end_time_ms: options.latencyMs ?? 200, + model_calls: 1, + discovery_calls: mode === "progressive" ? 1 : 0, + }, + }; +} + +function completeEvidence(progressiveSchemaTokens = 10): ToolDisclosureRun[] { + const runs: ToolDisclosureRun[] = []; + for (const campaign of CAMPAIGNS) { + for (const agent of AGENTS) { + runs.push( + makeRun({ + campaign, + agent, + mode: "direct", + phase: "static-visibility", + taskId: "static", + schemaTokens: 1_000, + visibleTools: 512, + }), + makeRun({ + campaign, + agent, + mode: "progressive", + phase: "static-visibility", + taskId: "static", + schemaTokens: progressiveSchemaTokens, + visibleTools: 3, + }), + ); + for (const taskId of TASK_IDS) { + runs.push( + makeRun({ + campaign, + agent, + mode: "direct", + taskId, + schemaTokens: 1_000, + latencyMs: 200, + }), + makeRun({ + campaign, + agent, + mode: "progressive", + taskId, + schemaTokens: progressiveSchemaTokens, + latencyMs: 100, + }), + ); + } + } + } + return runs; +} + +describe("tool-disclosure paired statistics", () => { + it("uses a deterministic 10,000-sample paired task bootstrap by default", () => { + const observations = [ + { direct: 10, progressive: 5 }, + { direct: 20, progressive: 16 }, + { direct: 30, progressive: 24 }, + ]; + const first = pairedBootstrapDifference(observations, { seed: 123 }); + const second = pairedBootstrapDifference(observations, { seed: 123 }); + + expect(first).toEqual(second); + expect(first.bootstrap_samples).toBe(DEFAULT_BOOTSTRAP_SAMPLES); + expect(first.paired_tasks).toBe(3); + expect(first.estimate).toBe(-5); + expect(first.lower_95).toBeLessThanOrEqual(first.estimate); + expect(first.upper_95).toBeGreaterThanOrEqual(first.estimate); + }); + + it("averages repetitions within a task before resampling task pairs", () => { + const runs = [ + makeRun({ mode: "direct", taskId: "task-a", repetition: 1, success: true }), + makeRun({ mode: "direct", taskId: "task-a", repetition: 2, success: false }), + makeRun({ mode: "progressive", taskId: "task-a", repetition: 1, success: true }), + makeRun({ mode: "progressive", taskId: "task-a", repetition: 2, success: true }), + makeRun({ mode: "direct", taskId: "task-b", success: true }), + makeRun({ mode: "progressive", taskId: "task-b", success: true }), + ]; + const [cell] = buildComparisonCells(runs, { samples: 100, seed: 1 }); + + expect(cell.differences.success_percentage_points.paired_tasks).toBe(2); + expect(cell.differences.success_percentage_points.estimate).toBe(25); + expect(cell.direct.success_rate_percent).toBe(75); + expect(cell.progressive.success_rate_percent).toBe(100); + }); +}); + +describe("tool-disclosure visibility and claim gates", () => { + it("rejects nondeterministic static visibility instead of rounding it", () => { + const runs = [ + makeRun({ phase: "static-visibility", mode: "direct", schemaTokens: 1_000 }), + makeRun({ phase: "static-visibility", mode: "direct", schemaTokens: 1_001 }), + makeRun({ phase: "static-visibility", mode: "progressive", schemaTokens: 10 }), + ]; + + expect(() => summarizeStaticVisibility(runs)).toThrow("nondeterministic"); + }); + + it("treats a success lower confidence bound of exactly -5 pp as noninferior", () => { + const runs = completeEvidence(); + const cells = buildComparisonCells(runs, { samples: 100, seed: 2 }); + const visibility = summarizeStaticVisibility(runs); + const target = cells.find( + (cell) => cell.campaign_id === "campaign-1" && cell.agent === "langchain-deepagents-code", + ); + if (!target) throw new Error("missing test cell"); + target.differences.success_percentage_points.lower_95 = -5; + + const gates = evaluateClaimGates(cells, visibility, { + agents: ["langchain-deepagents-code"], + campaignIds: CAMPAIGNS, + primaryCatalogSize: 512, + noninferiorityMarginPercentagePoints: -5, + }); + + expect(gates.campaign_agent_gates[0].success_noninferior).toBe(true); + expect(gates.cross_agent_success_noninferior).toBe(true); + }); + + it("blocks a cross-agent claim when the improvement does not repeat in campaign two", () => { + const runs = completeEvidence(); + const cells = buildComparisonCells(runs, { samples: 100, seed: 2 }); + const visibility = summarizeStaticVisibility(runs); + const failingCell = cells.find( + (cell) => cell.campaign_id === "campaign-2" && cell.agent === "openclaw", + ); + if (!failingCell) throw new Error("missing test cell"); + failingCell.differences.initial_tool_schema_tokens.upper_95 = 1; + + const gates = evaluateClaimGates(cells, visibility, { + agents: AGENTS, + campaignIds: CAMPAIGNS, + primaryCatalogSize: 512, + }); + + expect(gates.cross_agent_initial_schema_tokens_improved).toBe(false); + expect( + gates.campaign_agent_gates.find( + (gate) => gate.campaign_id === "campaign-2" && gate.agent === "openclaw", + )?.initial_schema_tokens_improved, + ).toBe(false); + }); + + it("does not trust a stale aggregate gate when a campaign-agent cell is blocked", () => { + const summary = buildToolDisclosureSummary(makeManifest(), completeEvidence(), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + summary.claim_gates.campaign_agent_gates[0].schema_tokens_upper_95 = 1; + + const claims = buildConservativeClaims(summary); + + expect(summary.claim_gates.cross_agent_initial_schema_tokens_improved).toBe(true); + expect(claims.some((claim) => claim.includes("initial serialized tool-schema tokens"))).toBe( + false, + ); + }); + + it("never says 99% unless every cited cell measures at least 99%", () => { + const below = buildToolDisclosureSummary(makeManifest(), completeEvidence(11), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + const atLeast = buildToolDisclosureSummary(makeManifest(), completeEvidence(10), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + const belowClaims = buildConservativeClaims(below); + const atLeastClaims = buildConservativeClaims(atLeast); + + atLeast.static_visibility[0].reduction.tokenizer_tokens_percent = 98.999; + const nearThresholdClaims = buildConservativeClaims(atLeast); + + expect(belowClaims.join(" ")).not.toContain("99%"); + expect(belowClaims[0]).toContain("initial serialized tool-schema tokens"); + expect(nearThresholdClaims.join(" ")).not.toContain("99%"); + expect(nearThresholdClaims[0]).toContain("98.9%"); + expect(atLeastClaims[0]).toContain("at least 99%"); + expect(atLeastClaims[0]).toContain("initial serialized tool-schema tokens"); + }); + + it("blocks schema and latency claims when task success is not noninferior", () => { + const runs = completeEvidence(); + for (const run of runs) { + if (run.phase === "primary" && run.mode === "progressive") { + run.outcome = "incorrect"; + run.correctness.task_success = false; + } + } + const summary = buildToolDisclosureSummary(makeManifest(), runs); + expect(summary.claims.join(" ")).not.toContain("reduced initial"); + expect(summary.claims.join(" ")).not.toContain("reduced end-to-end"); + }); +}); + +describe("tool-disclosure Markdown report", () => { + it("renders only gated claims and public-safe evidence fields", () => { + const runs = completeEvidence(); + (runs[0] as ToolDisclosureRun & { prompt: string }).prompt = "RAW-PROMPT-SECRET"; + const manifest = makeManifest(); + const summary = buildToolDisclosureSummary(manifest, runs, { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + const markdown = renderToolDisclosureMarkdown(manifest, summary, { + artifacts: [ + { + artifact_id: "summary", + kind: "summary", + file_name: "summary.json", + media_type: "application/json", + byte_length: 123, + sha256: "a".repeat(64), + }, + ], + }); + + expect(markdown).toContain("# Progressive Tool-Disclosure Benchmark"); + expect(markdown).toContain("at least 99%"); + expect(markdown).toContain("initial serialized tool-schema tokens"); + expect(markdown).toContain("not an authorization boundary"); + expect(markdown).toContain("summary.json"); + expect(markdown).not.toContain("RAW-PROMPT-SECRET"); + }); + + it("rejects artifact paths so host locations cannot enter the report", () => { + const manifest = makeManifest(); + const summary = buildToolDisclosureSummary(manifest, completeEvidence(), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + + expect(() => + renderToolDisclosureMarkdown(manifest, summary, { + artifacts: [ + { + artifact_id: "bad", + kind: "summary", + file_name: "/Users/test/summary.json", + media_type: "application/json", + byte_length: 1, + sha256: "b".repeat(64), + }, + ], + }), + ).toThrow("leaf file name"); + }); +}); diff --git a/test/bench/tool-disclosure-schedule.test.ts b/test/bench/tool-disclosure-schedule.test.ts new file mode 100644 index 0000000000..b003df57ff --- /dev/null +++ b/test/bench/tool-disclosure-schedule.test.ts @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + appendJsonLine, + ensureCampaignDirectory, + readJsonLines, + scanArtifactsForForbiddenValues, + writeChecksumManifest, + writeJsonArtifact, +} from "../../scripts/bench/tool-disclosure/artifacts"; +import { + buildToolDisclosureSchedule, + countScheduledRunsByCampaign, +} from "../../scripts/bench/tool-disclosure/schedule"; + +const primaryTaskIds = Array.from({ length: 24 }, (_, index) => `task-${index + 1}`); +const stressTaskIds = primaryTaskIds.slice(0, 8); + +describe("tool-disclosure schedule", () => { + it("builds the frozen two-campaign matrix with adjacent balanced mode pairs", () => { + const schedule = buildToolDisclosureSchedule({ primaryTaskIds, stressTaskIds, seed: 6251 }); + expect(schedule).toHaveLength(1_884); + expect(countScheduledRunsByCampaign(schedule)).toEqual({ + "campaign-1:static-visibility": 30, + "campaign-1:small-control": 144, + "campaign-1:primary": 720, + "campaign-1:large-stress": 48, + "campaign-2:static-visibility": 30, + "campaign-2:small-control": 144, + "campaign-2:primary": 720, + "campaign-2:large-stress": 48, + }); + for (let index = 0; index < schedule.length; index += 2) { + expect(schedule[index].pair_id).toBe(schedule[index + 1].pair_id); + expect(new Set([schedule[index].mode, schedule[index + 1].mode])).toEqual( + new Set(["progressive", "direct"]), + ); + } + }); + + it("is deterministic for a fixed seed and changes order for another seed", () => { + const first = buildToolDisclosureSchedule({ primaryTaskIds, stressTaskIds, seed: 7 }); + const second = buildToolDisclosureSchedule({ primaryTaskIds, stressTaskIds, seed: 7 }); + const other = buildToolDisclosureSchedule({ primaryTaskIds, stressTaskIds, seed: 8 }); + expect(second).toEqual(first); + expect(other.map((run) => run.run_id)).not.toEqual(first.map((run) => run.run_id)); + }); +}); + +describe("tool-disclosure artifacts", () => { + it("writes JSON, JSONL, checksums, and rejects leaked values", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-tool-bench-")); + try { + const output = ensureCampaignDirectory(path.join(root, "campaign"), false); + writeJsonArtifact(output, "manifest.json", { safe: true }); + appendJsonLine(output, "runs.jsonl", { run_id: "one" }); + appendJsonLine(output, "runs.jsonl", { run_id: "two" }); + expect(readJsonLines(path.join(output, "runs.jsonl"))).toEqual([ + { run_id: "one" }, + { run_id: "two" }, + ]); + writeChecksumManifest(output, ["runs.jsonl", "manifest.json"]); + expect(fs.readFileSync(path.join(output, "SHA256SUMS"), "utf8")).toContain("manifest.json"); + scanArtifactsForForbiddenValues(output, ["manifest.json", "runs.jsonl"], ["secret"]); + expect(() => scanArtifactsForForbiddenValues(output, ["runs.jsonl"], ["run_id"])).toThrow( + "contains a forbidden value", + ); + expect(() => ensureCampaignDirectory(output, false)).toThrow("not empty"); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/test/bench/tool-disclosure-telemetry.test.ts b/test/bench/tool-disclosure-telemetry.test.ts new file mode 100644 index 0000000000..d048b3eba5 --- /dev/null +++ b/test/bench/tool-disclosure-telemetry.test.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + countTokensWithVllm, + parseProcessStartTime, + parseVllmTokenMetrics, + tokenDelta, + validatedTelemetryUrl, +} from "../../scripts/bench/tool-disclosure/telemetry"; + +describe("vLLM benchmark telemetry", () => { + it("sums labelled token counters and computes monotonic deltas", () => { + const snapshot = parseVllmTokenMetrics(` +# HELP vllm:prompt_tokens_total prompt +vllm:prompt_tokens_total{model_name="bench",engine="0"} 100 +vllm:prompt_tokens_total{model_name="bench",engine="1"} 20 +vllm:generation_tokens_total{model_name="bench",engine="0"} 30 +vllm:generation_tokens_total{model_name="bench",engine="1"} 2 +`); + expect(snapshot).toEqual({ prompt_tokens: 120, generation_tokens: 32 }); + expect(parseProcessStartTime("process_start_time_seconds 1712345678\n")).toBe(1712345678); + expect( + tokenDelta( + { prompt_tokens: 100, generation_tokens: 30 }, + { prompt_tokens: 120, generation_tokens: 32 }, + ), + ).toEqual({ prompt_tokens: 20, generation_tokens: 2, available: true }); + expect( + tokenDelta( + { prompt_tokens: 120, generation_tokens: 32 }, + { prompt_tokens: 100, generation_tokens: 30 }, + ).available, + ).toBe(false); + }); + + it("rejects unsafe telemetry endpoints", () => { + expect(() => validatedTelemetryUrl("http://example.com:8000", "/metrics")).toThrow( + "only on loopback", + ); + expect(() => validatedTelemetryUrl("https://user:secret@example.com", "/metrics")).toThrow( + "must not contain credentials", + ); + expect(validatedTelemetryUrl("http://127.0.0.1:8000/v1", "/metrics").toString()).toBe( + "http://127.0.0.1:8000/metrics", + ); + }); + + it("reads token count variants without exposing response bodies", async () => { + const fetchImpl = (async () => + new Response(JSON.stringify({ tokens: [1, 2, 3, 4] }), { + status: 200, + headers: { "content-type": "application/json" }, + })) as typeof fetch; + await expect( + countTokensWithVllm("http://localhost:8000", "bench-model", "[]", fetchImpl), + ).resolves.toBe(4); + }); +}); diff --git a/test/bench/tool-disclosure-validation.test.ts b/test/bench/tool-disclosure-validation.test.ts new file mode 100644 index 0000000000..182549d166 --- /dev/null +++ b/test/bench/tool-disclosure-validation.test.ts @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeAll, describe, expect, it } from "vitest"; + +import { assembleToolDisclosureRun } from "../../scripts/bench/tool-disclosure/assemble-run"; +import { + canonicalJson, + generateSyntheticCatalog, + type SyntheticCatalog, + sha256Hex, +} from "../../scripts/bench/tool-disclosure/catalog"; +import type { SanitizedRunEvidence } from "../../scripts/bench/tool-disclosure/execute"; +import type { RecordedSyntheticCall } from "../../scripts/bench/tool-disclosure/grading"; +import type { ToolDisclosureRecordingEvent } from "../../scripts/bench/tool-disclosure/recorder"; +import { + buildToolDisclosureSchedule, + type ScheduledToolDisclosureRun, + STATIC_CATALOG_SIZES, + TOOL_DISCLOSURE_AGENTS, + TOOL_DISCLOSURE_MODES, +} from "../../scripts/bench/tool-disclosure/schedule"; +import { + generatePrimaryTaskSet, + generateStressTaskSet, + type SyntheticBenchmarkTask, + type SyntheticTaskSet, +} from "../../scripts/bench/tool-disclosure/tasks"; +import { + DEFAULT_BOOTSTRAP_SAMPLES, + DEFAULT_BOOTSTRAP_SEED, + DEFAULT_NONINFERIORITY_MARGIN_PP, + TOOL_DISCLOSURE_SCHEMA_VERSION, + type ToolDisclosureManifest, + type ToolDisclosureRun, +} from "../../scripts/bench/tool-disclosure/types"; +import { + validateCompleteEvidence, + validateFrozenManifest, +} from "../../scripts/bench/tool-disclosure/validation"; + +interface CompleteEvidenceFixture { + catalog: SyntheticCatalog; + manifest: ToolDisclosureManifest; + primaryTasks: SyntheticTaskSet; + rawEvidence: SanitizedRunEvidence[]; + runs: ToolDisclosureRun[]; + schedule: ScheduledToolDisclosureRun[]; + stressTasks: SyntheticTaskSet; +} + +function buildManifest( + primaryTasks: SyntheticTaskSet, + stressTasks: SyntheticTaskSet, +): ToolDisclosureManifest { + const allTasks = [...primaryTasks.tasks, ...stressTasks.tasks]; + const sandboxImageDigests = Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + `sha256:${"a".repeat(64)}`, + ]), + ), + ), + ); + return { + schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, + benchmark_id: "tool-disclosure-complete-evidence-fixture", + created_at: "2026-07-06T12:00:00.000Z", + sut: { git_sha: "1".repeat(40), git_ref: "fixture-sut", worktree_clean: true }, + harness: { git_sha: "2".repeat(40), git_ref: "fixture-harness", worktree_clean: true }, + campaigns: [ + { + campaign_id: "campaign-1", + ordinal: 1, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + { + campaign_id: "campaign-2", + ordinal: 2, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + ], + protocol: { + agents: TOOL_DISCLOSURE_AGENTS, + modes: TOOL_DISCLOSURE_MODES, + catalog_sizes: STATIC_CATALOG_SIZES, + primary_catalog_size: 512, + tasks: allTasks.map((task) => ({ task_id: task.id, kind: task.kind })), + repetitions: { "small-control": 1, primary: 5, "large-stress": 1 }, + execution_seed: DEFAULT_BOOTSTRAP_SEED, + bootstrap_samples: DEFAULT_BOOTSTRAP_SAMPLES, + bootstrap_seed: DEFAULT_BOOTSTRAP_SEED, + noninferiority_margin_percentage_points: DEFAULT_NONINFERIORITY_MARGIN_PP, + retry_setup_failures: 1, + }, + environment: { + operating_system: "fixture-os", + architecture: "x86_64", + cpu_model: "fixture-cpu", + cpu_count: 1, + ram_gib: 1, + gpu_model: "fixture-gpu", + gpu_architecture: "fixture-architecture", + gpu_count: 1, + gpu_driver_version: "fixture-driver", + cuda_version: "fixture-cuda", + power_state: "fixture-power-state", + openshell_version: "fixture-openshell", + agent_versions: { + openclaw: "fixture-openclaw", + hermes: "fixture-hermes", + "langchain-deepagents-code": "fixture-deepagents", + }, + sandbox_image_digests: sandboxImageDigests, + }, + inference: { + api: "chat-completions", + model_id: "fixture-model", + model_revision: "fixture-model-revision", + container_image: "fixture-inference-image", + container_digest: `sha256:${"b".repeat(64)}`, + vllm_version: "fixture-vllm", + tool_call_parser: "fixture-tool-parser", + reasoning_parser: "fixture-reasoning-parser", + temperature: 0, + concurrency: 1, + prefix_caching_enabled: false, + public_vllm_flags: [], + }, + }; +} + +function recordedCalls(task: SyntheticBenchmarkTask | undefined): RecordedSyntheticCall[] { + return ( + task?.expected_calls.map((call) => ({ + tool_name: call.tool_name, + arguments_sha256: sha256Hex(canonicalJson(call.arguments)), + result_nonce: call.result_nonce, + success: true, + })) ?? [] + ); +} + +function recordingEvent(options: { + run: ScheduledToolDisclosureRun; + toolNames: readonly string[]; +}): ToolDisclosureRecordingEvent { + return { + run_id: options.run.run_id, + request_sequence: 1, + model_call_sequence: 1, + endpoint: "chat-completions", + method: "POST", + visible_tool_count: options.toolNames.length, + canonical_tools_json_bytes: Math.max(1, options.toolNames.length * 64), + tools_sha256: sha256Hex(options.toolNames.join("\0")), + tool_names: options.toolNames, + streaming: true, + status_code: 200, + started_monotonic_ms: 1, + first_byte_monotonic_ms: 2, + ended_monotonic_ms: 3, + duration_ms: 2, + time_to_first_byte_ms: 1, + outcome: "completed", + error_reason: null, + }; +} + +function buildCompleteEvidenceFixture(): CompleteEvidenceFixture { + const catalog = generateSyntheticCatalog(); + const primaryTasks = generatePrimaryTaskSet(catalog); + const stressTasks = generateStressTaskSet(catalog); + const manifest = buildManifest(primaryTasks, stressTasks); + const schedule = buildToolDisclosureSchedule({ + primaryTaskIds: primaryTasks.tasks.map((task) => task.id), + stressTaskIds: stressTasks.tasks.map((task) => task.id), + seed: manifest.protocol.execution_seed, + }); + const tasksById = new Map( + [...primaryTasks.tasks, ...stressTasks.tasks].map((task) => [task.id, task] as const), + ); + const directToolNamesBySize: ReadonlyMap = new Map( + STATIC_CATALOG_SIZES.map((size) => [ + size, + catalog.tools.slice(0, size).map((tool) => tool.definition.function.name), + ]), + ); + const progressiveToolNames = ["search_tools"] as const; + const rawEvidence: SanitizedRunEvidence[] = []; + const runs: ToolDisclosureRun[] = []; + + for (const scheduled of schedule) { + const task = scheduled.task_id ? tasksById.get(scheduled.task_id) : undefined; + const calls = recordedCalls(task); + const toolNames = + scheduled.mode === "direct" + ? directToolNamesBySize.get(scheduled.catalog_size) + : progressiveToolNames; + if (!toolNames) throw new Error(`missing tool-name fixture for ${scheduled.catalog_size}`); + const recorderEvents = [recordingEvent({ run: scheduled, toolNames })]; + const initialSchemaTokens = scheduled.mode === "direct" ? scheduled.catalog_size * 8 : 8; + const promptTokens = initialSchemaTokens + 100; + const completionTokens = 16; + const invocation = { exit_code: 0, timed_out: false, elapsed_ms: 10 }; + rawEvidence.push({ + run_id: scheduled.run_id, + recorder_events: recorderEvents, + calls, + invocation, + initial_schema_tokens: initialSchemaTokens, + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + final_oracles_present: Boolean(task), + }); + runs.push( + assembleToolDisclosureRun({ + manifest, + scheduled, + task, + calls, + recorderEvents, + invocation: { + ...invocation, + final_output: task?.expected_final_includes.join(" ") ?? "", + }, + initialSchemaTokens, + promptTokens, + completionTokens, + }), + ); + } + + return { catalog, manifest, primaryTasks, rawEvidence, runs, schedule, stressTasks }; +} + +describe("tool-disclosure complete-evidence validation", () => { + let fixture: CompleteEvidenceFixture; + + beforeAll(() => { + fixture = buildCompleteEvidenceFixture(); + }); + + it("rejects incomplete or mutable claim-grade manifest metadata", () => { + expect(() => validateFrozenManifest(fixture.manifest)).not.toThrow(); + + const missingModelRevision = structuredClone(fixture.manifest); + missingModelRevision.inference.model_revision = ""; + expect(() => validateFrozenManifest(missingModelRevision)).toThrow(/claim-grade/u); + + const mutableInferenceImage = structuredClone(fixture.manifest); + mutableInferenceImage.inference.container_digest = "fixture:latest"; + expect(() => validateFrozenManifest(mutableInferenceImage)).toThrow(/claim-grade/u); + }); + + it("accepts the exact 1,884-run frozen matrix and rejects reordered or tampered evidence", { + timeout: 30_000, + }, () => { + expect(fixture.schedule).toHaveLength(1_884); + expect(fixture.runs).toHaveLength(1_884); + expect(() => validateCompleteEvidence(fixture)).not.toThrow(); + + const reorderedRuns = [...fixture.runs]; + const reorderedRaw = [...fixture.rawEvidence]; + [reorderedRuns[0], reorderedRuns[1]] = [reorderedRuns[1], reorderedRuns[0]]; + [reorderedRaw[0], reorderedRaw[1]] = [reorderedRaw[1], reorderedRaw[0]]; + expect(() => + validateCompleteEvidence({ + ...fixture, + runs: reorderedRuns, + rawEvidence: reorderedRaw, + }), + ).toThrow("runs.jsonl is out of frozen schedule order"); + + const rawOnlyReordered = [...fixture.rawEvidence]; + [rawOnlyReordered[0], rawOnlyReordered[1]] = [rawOnlyReordered[1], rawOnlyReordered[0]]; + expect(() => validateCompleteEvidence({ ...fixture, rawEvidence: rawOnlyReordered })).toThrow( + "raw evidence order differs", + ); + + const tamperedRaw = [...fixture.rawEvidence]; + tamperedRaw[0] = { + ...tamperedRaw[0], + initial_schema_tokens: tamperedRaw[0].initial_schema_tokens + 1, + }; + expect(() => validateCompleteEvidence({ ...fixture, rawEvidence: tamperedRaw })).toThrow( + "does not match raw evidence", + ); + }); +}); From ff038cb54c48edd1167e84e9bca28a496aca1e1e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 13:56:57 -0700 Subject: [PATCH 02/31] test(bench): keep disclosure fixtures linear Signed-off-by: Aaron Erickson --- test/bench/tool-disclosure-catalog.test.ts | 14 ++++---- test/bench/tool-disclosure-fixture.test.ts | 35 +++++++++++++------ test/bench/tool-disclosure-recorder.test.ts | 14 ++++---- test/bench/tool-disclosure-report.test.ts | 20 ++++++----- test/bench/tool-disclosure-validation.test.ts | 5 +-- 5 files changed, 52 insertions(+), 36 deletions(-) diff --git a/test/bench/tool-disclosure-catalog.test.ts b/test/bench/tool-disclosure-catalog.test.ts index ad54218bb5..95ea33473c 100644 --- a/test/bench/tool-disclosure-catalog.test.ts +++ b/test/bench/tool-disclosure-catalog.test.ts @@ -91,9 +91,9 @@ describe("synthetic tool-disclosure catalog", () => { ]; for (const tool of samples) { expect(tool).toBeDefined(); - if (!tool) continue; - const args = buildSyntheticArguments(tool, 17); - expect(validateSyntheticArguments(tool, args)).toEqual([]); + const requiredTool = tool as NonNullable; + const args = buildSyntheticArguments(requiredTool, 17); + expect(validateSyntheticArguments(requiredTool, args)).toEqual([]); } expect(Object.keys(samples[0]?.definition.function.parameters.properties ?? {})).toHaveLength( 2, @@ -170,12 +170,12 @@ describe("canonical fixture execution", () => { it("rejects malformed handler arguments before producing a result", () => { const tool = fullCatalog.tools.find((candidate) => candidate.complexity === "large"); expect(tool).toBeDefined(); - if (!tool) return; - const valid = buildSyntheticArguments(tool, 3); - expect(validateSyntheticArguments(tool, { ...valid, unexpected: true })).toContain( + const requiredTool = tool as NonNullable; + const valid = buildSyntheticArguments(requiredTool, 3); + expect(validateSyntheticArguments(requiredTool, { ...valid, unexpected: true })).toContain( "arguments.unexpected is not allowed", ); - expect(() => executeSyntheticTool(tool, { ...valid, limit: 0 })).toThrow( + expect(() => executeSyntheticTool(requiredTool, { ...valid, limit: 0 })).toThrow( SyntheticToolInvocationError, ); }); diff --git a/test/bench/tool-disclosure-fixture.test.ts b/test/bench/tool-disclosure-fixture.test.ts index ef1ec6aff3..e3e99cd025 100644 --- a/test/bench/tool-disclosure-fixture.test.ts +++ b/test/bench/tool-disclosure-fixture.test.ts @@ -79,17 +79,26 @@ describe("benchmark grading", () => { it("requires exact ordered calls, argument hashes, nonces, and final oracle", () => { const catalog = generateSyntheticCatalog({ size: 64 }); const task = buildPrimaryTasks(catalog).find((candidate) => candidate.kind === "ordered-chain"); - if (!task) throw new Error("chain task fixture missing"); - const calls = task.expected_calls.map((call) => ({ + expect(task).toBeDefined(); + const requiredTask = task as NonNullable; + const calls = requiredTask.expected_calls.map((call) => ({ tool_name: call.tool_name, arguments_sha256: sha256Hex(canonicalJson(call.arguments)), result_nonce: call.result_nonce, success: true, })); - const passing = gradeTaskRun(task, calls, task.expected_final_includes.join(" ")); + const passing = gradeTaskRun( + requiredTask, + calls, + requiredTask.expected_final_includes.join(" "), + ); expect(passing.outcome).toBe("success"); expect( - gradeTaskRun(task, [...calls].reverse(), task.expected_final_includes.join(" ")), + gradeTaskRun( + requiredTask, + [...calls].reverse(), + requiredTask.expected_final_includes.join(" "), + ), ).toMatchObject({ outcome: "incorrect", correctness: { expected_tool_order: false } }); }); @@ -97,11 +106,14 @@ describe("benchmark grading", () => { const task = buildPrimaryTasks(generateSyntheticCatalog({ size: 64 })).find( (candidate) => candidate.kind === "no-tool", ); - if (!task) throw new Error("no-tool fixture missing"); - expect(gradeTaskRun(task, [], task.expected_final_includes[0]).outcome).toBe("success"); + expect(task).toBeDefined(); + const requiredTask = task as NonNullable; + expect(gradeTaskRun(requiredTask, [], requiredTask.expected_final_includes[0]).outcome).toBe( + "success", + ); expect( gradeTaskRun( - task, + requiredTask, [ { tool_name: "unexpected", @@ -110,7 +122,7 @@ describe("benchmark grading", () => { success: true, }, ], - task.expected_final_includes[0], + requiredTask.expected_final_includes[0], ).outcome, ).toBe("incorrect"); }); @@ -125,7 +137,8 @@ describe("benchmark grading", () => { seed: 1, campaigns: [1], }).find((run) => run.task_id === task.id && run.agent === "openclaw"); - if (!scheduled) throw new Error("scheduled fixture missing"); + expect(scheduled).toBeDefined(); + const requiredScheduled = scheduled as NonNullable; const manifest = { schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, benchmark_id: "fixture-benchmark", @@ -142,7 +155,7 @@ describe("benchmark grading", () => { const expected = task.expected_calls[0]; const record = assembleToolDisclosureRun({ manifest, - scheduled, + scheduled: requiredScheduled, task, calls: [ { @@ -154,7 +167,7 @@ describe("benchmark grading", () => { ], recorderEvents: [ { - run_id: scheduled.run_id, + run_id: requiredScheduled.run_id, request_sequence: 1, model_call_sequence: 1, endpoint: "chat-completions", diff --git a/test/bench/tool-disclosure-recorder.test.ts b/test/bench/tool-disclosure-recorder.test.ts index 0ef2736cf4..3ee67ec716 100644 --- a/test/bench/tool-disclosure-recorder.test.ts +++ b/test/bench/tool-disclosure-recorder.test.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; import http, { type IncomingMessage, type ServerResponse } from "node:http"; -import type { Socket } from "node:net"; +import type { AddressInfo, Socket } from "node:net"; import { afterEach, describe, expect, it } from "vitest"; @@ -37,14 +37,16 @@ async function startUpstream( server.listen(0, "127.0.0.1", () => resolve()); }); const address = server.address(); - if (!address || typeof address === "string") throw new Error("test upstream did not bind"); + expect(address).not.toBeNull(); + expect(typeof address).not.toBe("string"); + const tcpAddress = address as AddressInfo; const close = async (): Promise => { for (const socket of sockets) socket.destroy(); await new Promise((resolve) => server.close(() => resolve())); }; cleanup.push(close); - return { baseUrl: `http://127.0.0.1:${address.port}`, close }; + return { baseUrl: `http://127.0.0.1:${tcpAddress.port}`, close }; } async function startProxy( @@ -380,10 +382,8 @@ describe("tool-disclosure recording proxy", () => { const upstream = await startUpstream(async (request, response) => { await collectBody(request); setTimeout(() => { - if (!response.destroyed) { - response.writeHead(500, { "content-type": "text/plain" }); - response.end("upstream-error-body-secret"); - } + response.writeHead(500, { "content-type": "text/plain" }); + response.end("upstream-error-body-secret"); }, 200); }); const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { diff --git a/test/bench/tool-disclosure-report.test.ts b/test/bench/tool-disclosure-report.test.ts index 406d2af611..ddb7c4eec7 100644 --- a/test/bench/tool-disclosure-report.test.ts +++ b/test/bench/tool-disclosure-report.test.ts @@ -273,8 +273,9 @@ describe("tool-disclosure visibility and claim gates", () => { const target = cells.find( (cell) => cell.campaign_id === "campaign-1" && cell.agent === "langchain-deepagents-code", ); - if (!target) throw new Error("missing test cell"); - target.differences.success_percentage_points.lower_95 = -5; + expect(target).toBeDefined(); + const requiredTarget = target as NonNullable; + requiredTarget.differences.success_percentage_points.lower_95 = -5; const gates = evaluateClaimGates(cells, visibility, { agents: ["langchain-deepagents-code"], @@ -294,8 +295,9 @@ describe("tool-disclosure visibility and claim gates", () => { const failingCell = cells.find( (cell) => cell.campaign_id === "campaign-2" && cell.agent === "openclaw", ); - if (!failingCell) throw new Error("missing test cell"); - failingCell.differences.initial_tool_schema_tokens.upper_95 = 1; + expect(failingCell).toBeDefined(); + const requiredFailingCell = failingCell as NonNullable; + requiredFailingCell.differences.initial_tool_schema_tokens.upper_95 = 1; const gates = evaluateClaimGates(cells, visibility, { agents: AGENTS, @@ -348,11 +350,11 @@ describe("tool-disclosure visibility and claim gates", () => { it("blocks schema and latency claims when task success is not noninferior", () => { const runs = completeEvidence(); - for (const run of runs) { - if (run.phase === "primary" && run.mode === "progressive") { - run.outcome = "incorrect"; - run.correctness.task_success = false; - } + for (const run of runs.filter( + (candidate) => candidate.phase === "primary" && candidate.mode === "progressive", + )) { + run.outcome = "incorrect"; + run.correctness.task_success = false; } const summary = buildToolDisclosureSummary(makeManifest(), runs); expect(summary.claims.join(" ")).not.toContain("reduced initial"); diff --git a/test/bench/tool-disclosure-validation.test.ts b/test/bench/tool-disclosure-validation.test.ts index 182549d166..a296f25f40 100644 --- a/test/bench/tool-disclosure-validation.test.ts +++ b/test/bench/tool-disclosure-validation.test.ts @@ -201,8 +201,9 @@ function buildCompleteEvidenceFixture(): CompleteEvidenceFixture { scheduled.mode === "direct" ? directToolNamesBySize.get(scheduled.catalog_size) : progressiveToolNames; - if (!toolNames) throw new Error(`missing tool-name fixture for ${scheduled.catalog_size}`); - const recorderEvents = [recordingEvent({ run: scheduled, toolNames })]; + expect(toolNames).toBeDefined(); + const requiredToolNames = toolNames as NonNullable; + const recorderEvents = [recordingEvent({ run: scheduled, toolNames: requiredToolNames })]; const initialSchemaTokens = scheduled.mode === "direct" ? scheduled.catalog_size * 8 : 8; const promptTokens = initialSchemaTokens + 100; const completionTokens = 16; From 8221385c83a1c40df2bcb4c0ac47bad184ff8a47 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 14:50:25 -0700 Subject: [PATCH 03/31] fix(bench): harden benchmark evidence boundaries Signed-off-by: Aaron Erickson --- .../progressive-tool-disclosure-benchmark.mdx | 1 + scripts/bench/tool-disclosure/assemble-run.ts | 13 +- scripts/bench/tool-disclosure/execute.ts | 48 +++++-- .../bench/tool-disclosure/openclaw-fixture.ts | 13 ++ scripts/bench/tool-disclosure/quick-tunnel.ts | 2 + scripts/bench/tool-disclosure/run.ts | 8 +- scripts/bench/tool-disclosure/schedule.ts | 47 +++++- scripts/bench/tool-disclosure/subprocess.ts | 8 +- test/bench/tool-disclosure-execute.test.ts | 22 +++ test/bench/tool-disclosure-fixture.test.ts | 135 +++++++++++++----- test/bench/tool-disclosure-mcp-server.test.ts | 6 + 11 files changed, 236 insertions(+), 67 deletions(-) diff --git a/docs/inference/progressive-tool-disclosure-benchmark.mdx b/docs/inference/progressive-tool-disclosure-benchmark.mdx index 8a88b88aee..456e2ed91e 100644 --- a/docs/inference/progressive-tool-disclosure-benchmark.mdx +++ b/docs/inference/progressive-tool-disclosure-benchmark.mdx @@ -153,6 +153,7 @@ If any of those controlled values changes, prepare a new benchmark ID and restar The agent adapters must use the generated deterministic handlers and verify expected tool names, order, argument hashes, call count, unnecessary calls, and the returned nonce. Direct and progressive modes must differ only in their disclosure configuration. Follow the retry allowance in `manifest.json` for setup failures, and do not convert task failures into setup failures. +Once an agent invocation has begun, any incomplete or inconsistent evidence aborts the campaign instead of consuming a setup retry; discard that campaign and restart it with fresh resources. ## Summarize and verify artifacts diff --git a/scripts/bench/tool-disclosure/assemble-run.ts b/scripts/bench/tool-disclosure/assemble-run.ts index f3404a15a7..7a73b77516 100644 --- a/scripts/bench/tool-disclosure/assemble-run.ts +++ b/scripts/bench/tool-disclosure/assemble-run.ts @@ -81,19 +81,18 @@ export function assembleToolDisclosureRun(options: { if (options.failureOutcome) { outcome = options.failureOutcome; correctness = failedCorrectness(options.calls.length); - } else if ( - options.scheduled.phase === "static-visibility" && - initial && - options.initialSchemaTokens > 0 - ) { - outcome = "success"; - correctness = successfulStaticCorrectness(); } else if (options.invocation.timed_out) { outcome = "timeout"; correctness = failedCorrectness(options.calls.length); } else if (options.invocation.exit_code !== 0 || !initial || initial.outcome !== "completed") { outcome = "model-error"; correctness = failedCorrectness(options.calls.length); + } else if (options.scheduled.phase === "static-visibility") { + outcome = options.initialSchemaTokens > 0 ? "success" : "model-error"; + correctness = + options.initialSchemaTokens > 0 + ? successfulStaticCorrectness() + : failedCorrectness(options.calls.length); } else { const graded = gradeTaskRun( options.task as SyntheticBenchmarkTask, diff --git a/scripts/bench/tool-disclosure/execute.ts b/scripts/bench/tool-disclosure/execute.ts index 4f0b40431d..0dd58377f6 100644 --- a/scripts/bench/tool-disclosure/execute.ts +++ b/scripts/bench/tool-disclosure/execute.ts @@ -44,10 +44,25 @@ export interface AttemptJournalEntry { run: ToolDisclosureRun; } +/** Permit retries only while an attempt is still wholly inside setup. */ +export function assertSetupRetryAllowed( + invocationAttempted: boolean, + runId: string, + cause?: unknown, +): void { + if (invocationAttempted) { + throw new Error( + `run ${runId} failed after agent invocation began; discard this campaign and restart it with fresh resources`, + { cause }, + ); + } +} + export function recoverAttemptJournal(outputDir: string): AttemptJournalEntry[] { const file = artifactPath(outputDir, "attempt-journal.jsonl"); if (!fs.existsSync(file)) return []; - const lines = fs.readFileSync(file, "utf8").split("\n"); + const raw = fs.readFileSync(file, "utf8"); + const lines = raw.split("\n"); const entries: AttemptJournalEntry[] = []; for (let index = 0; index < lines.length; index += 1) { const line = lines[index]; @@ -55,8 +70,10 @@ export function recoverAttemptJournal(outputDir: string): AttemptJournalEntry[] try { entries.push(JSON.parse(line) as AttemptJournalEntry); } catch { - const isTrailing = lines.slice(index + 1).every((candidate) => !candidate.trim()); - if (!isTrailing) throw new Error(`attempt journal is corrupt at line ${index + 1}`); + const isUnterminatedFinalAppend = index === lines.length - 1 && !raw.endsWith("\n"); + if (!isUnterminatedFinalAppend) { + throw new Error(`attempt journal is corrupt at line ${index + 1}`); + } writeTextArtifact( outputDir, "attempt-journal.jsonl", @@ -161,6 +178,7 @@ async function attestVllmContainer(options: { ), }); if ( + containers.length !== 1 || !container || container.Id !== options.config.vllm_container_id || container.Image !== expected.container_digest || @@ -224,6 +242,7 @@ async function attestSandbox(options: { }>; const container = containers[0]; if ( + containers.length !== 1 || !container || container.Id !== instanceId || container.Image !== imageDigest || @@ -479,6 +498,7 @@ export async function executeCampaign(options: { attempt <= options.manifest.protocol.retry_setup_failures; attempt += 1 ) { + let invocationAttempted = false; try { if (run.agent === "openclaw") { const reset = await command( @@ -495,16 +515,15 @@ export async function executeCampaign(options: { mcp?.beginRun(run.run_id); proxy.beginRun(run.run_id); const before = await readVllmTokenSnapshot(options.config.telemetry_url); - const invocation = await command( - buildAgentDriverCommand({ - openshellBin: options.config.openshell_bin, - sandboxName, - agent: run.agent, - prompt: task?.prompt ?? "Without using tools, reply exactly STATIC-CAPTURE.", - sessionId: run.run_id, - }), - timeoutMs, - ); + const driverCommand = buildAgentDriverCommand({ + openshellBin: options.config.openshell_bin, + sandboxName, + agent: run.agent, + prompt: task?.prompt ?? "Without using tools, reply exactly STATIC-CAPTURE.", + sessionId: run.run_id, + }); + invocationAttempted = true; + const invocation = await command(driverCommand, timeoutMs); if (invocation.output_truncated) throw new Error("agent output exceeded the evidence bound"); const after = await readVllmTokenSnapshot(options.config.telemetry_url); @@ -581,13 +600,14 @@ export async function executeCampaign(options: { appendJsonLine(options.outputDir, "attempt-journal.jsonl", entry); journal.push(entry); break; - } catch { + } catch (error) { try { proxy.endRun(); } catch {} try { mcp?.endRun(); } catch {} + assertSetupRetryAllowed(invocationAttempted, run.run_id, error); const setupEvidence: SanitizedRunEvidence = { run_id: run.run_id, recorder_events: [], diff --git a/scripts/bench/tool-disclosure/openclaw-fixture.ts b/scripts/bench/tool-disclosure/openclaw-fixture.ts index 76157bfb3f..a8a3c4ecd6 100644 --- a/scripts/bench/tool-disclosure/openclaw-fixture.ts +++ b/scripts/bench/tool-disclosure/openclaw-fixture.ts @@ -7,6 +7,9 @@ import path from "node:path"; import type { SyntheticCatalog } from "./catalog"; const SAFE_FIXTURE_DIR = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u; +const SAFE_IMMUTABLE_SANDBOX_BASE = + /^[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]{1,5})?(?:\/[a-z0-9]+(?:[._-][a-z0-9]+)*)+@sha256:[a-f0-9]{64}$/u; +const WHITESPACE_OR_CONTROL = /[\s\u0000-\u001f\u007f]/u; export interface OpenClawFixtureOptions { outputDir: string; @@ -26,6 +29,15 @@ function writeNewFile(directory: string, name: string, content: string): void { fs.writeFileSync(destination, content, { encoding: "utf8", flag: "wx", mode: 0o600 }); } +/** Reject mutable or shell/Dockerfile-active image references before writing a fixture. */ +export function assertImmutableSandboxBase(image: string): void { + if (WHITESPACE_OR_CONTROL.test(image) || !SAFE_IMMUTABLE_SANDBOX_BASE.test(image)) { + throw new Error( + "sandbox base must be a canonical registry/repository reference with an immutable sha256 digest", + ); + } +} + function pluginSource(): string { return `// Generated by the NemoClaw progressive tool-disclosure benchmark.\n\ import { createHash } from "node:crypto";\n\ @@ -103,6 +115,7 @@ WORKDIR /opt/nemoclaw\n`; /** Generate a self-contained custom-image context for an OpenClaw catalog size. */ export function writeOpenClawFixture(options: OpenClawFixtureOptions): OpenClawFixtureResult { + assertImmutableSandboxBase(options.sandboxBase); const root = path.resolve(options.outputDir); if (fs.existsSync(root)) { const stat = fs.lstatSync(root); diff --git a/scripts/bench/tool-disclosure/quick-tunnel.ts b/scripts/bench/tool-disclosure/quick-tunnel.ts index b05e60374c..ef4e140720 100644 --- a/scripts/bench/tool-disclosure/quick-tunnel.ts +++ b/scripts/bench/tool-disclosure/quick-tunnel.ts @@ -4,6 +4,8 @@ import { type ChildProcess, spawn } from "node:child_process"; const ORIGIN_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com(?=$|[\s"'\\/])/iu; +// Deliberately omit HOME/XDG_CONFIG_HOME so cloudflared cannot load ambient +// account credentials or configuration into this ephemeral benchmark tunnel. const SAFE_ENV = new Set([ "PATH", "TMPDIR", diff --git a/scripts/bench/tool-disclosure/run.ts b/scripts/bench/tool-disclosure/run.ts index e6f92f5168..49c9ee4805 100644 --- a/scripts/bench/tool-disclosure/run.ts +++ b/scripts/bench/tool-disclosure/run.ts @@ -30,7 +30,7 @@ import { type LiveCampaignConfiguration, type SanitizedRunEvidence, } from "./execute"; -import { writeOpenClawFixture } from "./openclaw-fixture"; +import { assertImmutableSandboxBase, writeOpenClawFixture } from "./openclaw-fixture"; import { renderToolDisclosureMarkdown } from "./report"; import { buildToolDisclosureSchedule, @@ -275,12 +275,10 @@ function prepare(options: ParsedArguments): void { "benchmark preparation requires a clean worktree; commit or stash changes first", ); } - if ( - !options.sandboxBase || - (!options.allowDirty && !/@sha256:[a-f0-9]{64}$/u.test(options.sandboxBase)) - ) { + if (!options.sandboxBase) { throw new Error("--sandbox-base is required and must end in an immutable @sha256 digest"); } + assertImmutableSandboxBase(options.sandboxBase); const outputDir = ensureCampaignDirectory(requestedOutput, options.resume); const catalog = generateSyntheticCatalog({ seed: options.catalogSeed }); const primary = generatePrimaryTaskSet(catalog); diff --git a/scripts/bench/tool-disclosure/schedule.ts b/scripts/bench/tool-disclosure/schedule.ts index 0743b72611..a588f3fbc7 100644 --- a/scripts/bench/tool-disclosure/schedule.ts +++ b/scripts/bench/tool-disclosure/schedule.ts @@ -1,10 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { PRIMARY_TASK_COUNT, STRESS_TASK_COUNT } from "./tasks"; + export const TOOL_DISCLOSURE_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; export const TOOL_DISCLOSURE_MODES = ["progressive", "direct"] as const; export const STATIC_CATALOG_SIZES = [16, 64, 256, 512, 2_209] as const; +const SMALL_CONTROL_CATALOG_SIZE = STATIC_CATALOG_SIZES[1]; +const PRIMARY_CATALOG_SIZE = STATIC_CATALOG_SIZES[3]; +const LARGE_STRESS_CATALOG_SIZE = STATIC_CATALOG_SIZES[4]; export type ToolDisclosureAgent = (typeof TOOL_DISCLOSURE_AGENTS)[number]; export type ToolDisclosureMode = (typeof TOOL_DISCLOSURE_MODES)[number]; @@ -125,14 +130,14 @@ export function buildToolDisclosureSchedule( ): ScheduledToolDisclosureRun[] { assertUniqueNonEmpty(options.primaryTaskIds, "primaryTaskIds"); assertUniqueNonEmpty(options.stressTaskIds, "stressTaskIds"); - if (options.primaryTaskIds.length !== 24) { + if (options.primaryTaskIds.length !== PRIMARY_TASK_COUNT) { throw new Error( - `primaryTaskIds must contain exactly 24 tasks, got ${options.primaryTaskIds.length}`, + `primaryTaskIds must contain exactly ${PRIMARY_TASK_COUNT} tasks, got ${options.primaryTaskIds.length}`, ); } - if (options.stressTaskIds.length !== 8) { + if (options.stressTaskIds.length !== STRESS_TASK_COUNT) { throw new Error( - `stressTaskIds must contain exactly 8 tasks, got ${options.stressTaskIds.length}`, + `stressTaskIds must contain exactly ${STRESS_TASK_COUNT} tasks, got ${options.stressTaskIds.length}`, ); } if (!Number.isSafeInteger(options.seed)) throw new Error("seed must be a safe integer"); @@ -153,15 +158,43 @@ export function buildToolDisclosureSchedule( ); } for (const taskId of options.primaryTaskIds) { - liveBlocks.push(makePair(campaign, "small-control", agent, 64, taskId, 1, options.seed)); + liveBlocks.push( + makePair( + campaign, + "small-control", + agent, + SMALL_CONTROL_CATALOG_SIZE, + taskId, + 1, + options.seed, + ), + ); for (let repetition = 1; repetition <= 5; repetition += 1) { liveBlocks.push( - makePair(campaign, "primary", agent, 512, taskId, repetition, options.seed), + makePair( + campaign, + "primary", + agent, + PRIMARY_CATALOG_SIZE, + taskId, + repetition, + options.seed, + ), ); } } for (const taskId of options.stressTaskIds) { - liveBlocks.push(makePair(campaign, "large-stress", agent, 2_209, taskId, 1, options.seed)); + liveBlocks.push( + makePair( + campaign, + "large-stress", + agent, + LARGE_STRESS_CATALOG_SIZE, + taskId, + 1, + options.seed, + ), + ); } } staticBlocks.sort( diff --git a/scripts/bench/tool-disclosure/subprocess.ts b/scripts/bench/tool-disclosure/subprocess.ts index 61044e62fc..291967c728 100644 --- a/scripts/bench/tool-disclosure/subprocess.ts +++ b/scripts/bench/tool-disclosure/subprocess.ts @@ -67,6 +67,7 @@ export async function runBoundedCommand(options: { }); let timedOut = false; + let hardStop: NodeJS.Timeout | undefined; const timer = setTimeout(() => { timedOut = true; try { @@ -75,7 +76,7 @@ export async function runBoundedCommand(options: { } catch { // Process already exited. } - const hardStop = setTimeout(() => { + hardStop = setTimeout(() => { try { if (child.pid && process.platform !== "win32") process.kill(-child.pid, "SIGKILL"); else child.kill("SIGKILL"); @@ -91,7 +92,10 @@ export async function runBoundedCommand(options: { child.once("error", reject); child.once("close", (code, signal) => resolve({ code, signal })); }, - ).finally(() => clearTimeout(timer)); + ).finally(() => { + clearTimeout(timer); + if (hardStop) clearTimeout(hardStop); + }); const redactions = options.redactions ?? []; return { exit_code: result.code, diff --git a/test/bench/tool-disclosure-execute.test.ts b/test/bench/tool-disclosure-execute.test.ts index ac3f230fef..27f02450ea 100644 --- a/test/bench/tool-disclosure-execute.test.ts +++ b/test/bench/tool-disclosure-execute.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { + assertSetupRetryAllowed, type AttemptJournalEntry, type CampaignAttestation, materializeAttemptJournal, @@ -59,6 +60,13 @@ function attestationFixture(): { } describe("tool-disclosure attempt journal", () => { + it("retries only failures that occur before agent invocation", () => { + expect(() => assertSetupRetryAllowed(false, "run-before-invocation")).not.toThrow(); + expect(() => + assertSetupRetryAllowed(true, "run-after-invocation", new Error("private detail")), + ).toThrow(/discard this campaign/u); + }); + it("recovers one trailing partial append and deterministically materializes evidence", () => { const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); try { @@ -89,6 +97,20 @@ describe("tool-disclosure attempt journal", () => { } }); + it("fails closed for a newline-terminated malformed journal record", () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); + try { + const file = path.join(output, "attempt-journal.jsonl"); + const malformed = '{"raw":\n'; + fs.writeFileSync(file, malformed); + + expect(() => recoverAttemptJournal(output)).toThrow(/corrupt at line 1/u); + expect(fs.readFileSync(file, "utf8")).toBe(malformed); + } finally { + fs.rmSync(output, { recursive: true, force: true }); + } + }); + it("requires two fresh campaigns with identical frozen inference configuration", () => { const { manifest, attestations } = attestationFixture(); expect(() => validateCampaignAttestations(manifest, attestations)).not.toThrow(); diff --git a/test/bench/tool-disclosure-fixture.test.ts b/test/bench/tool-disclosure-fixture.test.ts index e3e99cd025..ca6c3bbde6 100644 --- a/test/bench/tool-disclosure-fixture.test.ts +++ b/test/bench/tool-disclosure-fixture.test.ts @@ -39,7 +39,11 @@ describe("OpenClaw benchmark fixture", () => { temporaryDirectories.push(root); const output = path.join(root, "fixture-16"); const catalog = generateSyntheticCatalog({ size: 16 }); - writeOpenClawFixture({ outputDir: output, catalog, sandboxBase: "example/base@sha256:1234" }); + writeOpenClawFixture({ + outputDir: output, + catalog, + sandboxBase: `registry.example/base@sha256:${"1".repeat(64)}`, + }); const registered: Array> = []; const callsPath = path.join(root, "calls.jsonl"); @@ -73,6 +77,22 @@ describe("OpenClaw benchmark fixture", () => { ) as { contracts: { tools: string[] } }; expect(manifest.contracts.tools).toHaveLength(16); }); + + it("rejects Dockerfile injection before creating the fixture", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bench-fixture-")); + temporaryDirectories.push(root); + const output = path.join(root, "fixture-injected"); + const injected = `registry.example/base\nRUN touch /tmp/injected\n#@sha256:${"1".repeat(64)}`; + + expect(() => + writeOpenClawFixture({ + outputDir: output, + catalog: generateSyntheticCatalog({ size: 16 }), + sandboxBase: injected, + }), + ).toThrow(/canonical registry\/repository reference/u); + expect(fs.existsSync(output)).toBe(false); + }); }); describe("benchmark grading", () => { @@ -127,7 +147,7 @@ describe("benchmark grading", () => { ).toBe("incorrect"); }); - it("assembles a public run without retaining model output or arguments", () => { + it("assembles public-safe success and terminal run outcomes", () => { const catalog = generateSyntheticCatalog({ size: 64 }); const tasks = buildPrimaryTasks(catalog); const task = tasks[0]; @@ -153,40 +173,42 @@ describe("benchmark grading", () => { protocol: { execution_seed: 1 }, } as unknown as ToolDisclosureManifest; const expected = task.expected_calls[0]; + const recordedCalls = [ + { + tool_name: expected.tool_name, + arguments_sha256: sha256Hex(canonicalJson(expected.arguments)), + result_nonce: expected.result_nonce, + success: true, + }, + ] as const; + const recorderEvents = [ + { + run_id: requiredScheduled.run_id, + request_sequence: 1, + model_call_sequence: 1, + endpoint: "chat-completions", + method: "POST", + visible_tool_count: 4, + canonical_tools_json_bytes: 321, + tools_sha256: "a".repeat(64), + tool_names: [expected.tool_name], + streaming: true, + status_code: 200, + started_monotonic_ms: 1, + first_byte_monotonic_ms: 2, + ended_monotonic_ms: 3, + duration_ms: 2, + time_to_first_byte_ms: 1, + outcome: "completed", + error_reason: null, + }, + ] as const; const record = assembleToolDisclosureRun({ manifest, scheduled: requiredScheduled, task, - calls: [ - { - tool_name: expected.tool_name, - arguments_sha256: sha256Hex(canonicalJson(expected.arguments)), - result_nonce: expected.result_nonce, - success: true, - }, - ], - recorderEvents: [ - { - run_id: requiredScheduled.run_id, - request_sequence: 1, - model_call_sequence: 1, - endpoint: "chat-completions", - method: "POST", - visible_tool_count: 4, - canonical_tools_json_bytes: 321, - tools_sha256: "a".repeat(64), - tool_names: [expected.tool_name], - streaming: true, - status_code: 200, - started_monotonic_ms: 1, - first_byte_monotonic_ms: 2, - ended_monotonic_ms: 3, - duration_ms: 2, - time_to_first_byte_ms: 1, - outcome: "completed", - error_reason: null, - }, - ], + calls: recordedCalls, + recorderEvents, invocation: { exit_code: 0, timed_out: false, @@ -203,5 +225,54 @@ describe("benchmark grading", () => { }); expect(JSON.stringify(record)).not.toContain("private output"); expect(JSON.stringify(record)).not.toContain(String(expected.arguments.resource_id)); + + for (const scenario of [ + { + expected: "timeout", + invocation: { exit_code: null, timed_out: true, elapsed_ms: 10, final_output: "" }, + failureOutcome: undefined, + }, + { + expected: "model-error", + invocation: { exit_code: 7, timed_out: false, elapsed_ms: 10, final_output: "" }, + failureOutcome: undefined, + }, + { + expected: "context-overflow", + invocation: { exit_code: 7, timed_out: false, elapsed_ms: 10, final_output: "" }, + failureOutcome: "context-overflow", + }, + ] as const) { + const terminal = assembleToolDisclosureRun({ + manifest, + scheduled: requiredScheduled, + task, + calls: recordedCalls, + recorderEvents, + invocation: scenario.invocation, + initialSchemaTokens: 42, + ...(scenario.failureOutcome ? { failureOutcome: scenario.failureOutcome } : {}), + }); + expect(terminal.outcome).toBe(scenario.expected); + expect(terminal.scored).toBe(true); + } + + const staticScheduled = buildToolDisclosureSchedule({ + primaryTaskIds: tasks.map((item) => item.id), + stressTaskIds: Array.from({ length: 8 }, (_, index) => `stress-${index}`), + seed: 1, + campaigns: [1], + }).find((run) => run.phase === "static-visibility"); + expect(staticScheduled).toBeDefined(); + const requiredStatic = staticScheduled as NonNullable; + const staticFailure = assembleToolDisclosureRun({ + manifest, + scheduled: requiredStatic, + calls: [], + recorderEvents: [{ ...recorderEvents[0], run_id: requiredStatic.run_id }], + invocation: { exit_code: 0, timed_out: false, elapsed_ms: 10, final_output: "" }, + initialSchemaTokens: 0, + }); + expect(staticFailure.outcome).toBe("model-error"); }); }); diff --git a/test/bench/tool-disclosure-mcp-server.test.ts b/test/bench/tool-disclosure-mcp-server.test.ts index 9bf1f7a275..fbf5fd5a83 100644 --- a/test/bench/tool-disclosure-mcp-server.test.ts +++ b/test/bench/tool-disclosure-mcp-server.test.ts @@ -34,6 +34,12 @@ describe("synthetic benchmark MCP server", () => { try { const unauthorized = await fetch(address.local_url, { method: "POST", body: "{}" }); expect(unauthorized.status).toBe(401); + const staleCampaignToken = await rpc( + address.local_url, + "token-from-another-campaign", + "tools/list", + ); + expect(staleCampaignToken.status).toBe(401); const initialized = await rpc(address.local_url, token, "initialize", { protocolVersion: "2025-03-26", From 83d1e31dc8cffbd56753196c3463c92bd7b972f2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 14:56:33 -0700 Subject: [PATCH 04/31] fix(bench): preserve terminal failure semantics Signed-off-by: Aaron Erickson --- scripts/bench/tool-disclosure/drivers.ts | 6 ++--- scripts/bench/tool-disclosure/execute.ts | 29 +++++++++++++++++----- test/bench/tool-disclosure-drivers.test.ts | 18 ++++++++++++++ test/bench/tool-disclosure-execute.test.ts | 22 ++++++++++++++++ 4 files changed, 66 insertions(+), 9 deletions(-) diff --git a/scripts/bench/tool-disclosure/drivers.ts b/scripts/bench/tool-disclosure/drivers.ts index 6c9d59655b..51548f1778 100644 --- a/scripts/bench/tool-disclosure/drivers.ts +++ b/scripts/bench/tool-disclosure/drivers.ts @@ -30,14 +30,14 @@ function shellScriptFor(agent: ToolDisclosureAgent, prompt: string, sessionId: s "set -eu", `prompt=$(printf '%s' '${encodedPrompt}' | base64 -d)`, `exec openclaw agent --agent main --json --thinking off --session-id '${sessionId}' -m "$prompt"`, - ].join("; "); + ].join("\n"); } if (agent === "langchain-deepagents-code") { return [ "set -eu", `prompt=$(printf '%s' '${encodedPrompt}' | base64 -d)`, 'exec nemoclaw-start dcode -n "$prompt"', - ].join("; "); + ].join("\n"); } const payload = base64( JSON.stringify({ @@ -57,7 +57,7 @@ function shellScriptFor(agent: ToolDisclosureAgent, prompt: string, sessionId: s ' exec curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H \'Content-Type: application/json\' -H "Authorization: Bearer ${API_SERVER_KEY}" --data-binary "$payload"', "fi", "exec curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary \"$payload\"", - ].join("; "); + ].join("\n"); } export function buildAgentDriverCommand(options: { diff --git a/scripts/bench/tool-disclosure/execute.ts b/scripts/bench/tool-disclosure/execute.ts index 0dd58377f6..e77b54a994 100644 --- a/scripts/bench/tool-disclosure/execute.ts +++ b/scripts/bench/tool-disclosure/execute.ts @@ -306,6 +306,22 @@ function isContextOverflow(stdout: string, stderr: string): boolean { ); } +export function classifyInvocationFailure(options: { + phase: ScheduledToolDisclosureRun["phase"]; + exitCode: number | null; + timedOut: boolean; + stdout: string; + stderr: string; +}): "context-overflow" | undefined { + return options.phase !== "static-visibility" && + !options.timedOut && + options.exitCode !== null && + options.exitCode !== 0 && + isContextOverflow(options.stdout, options.stderr) + ? "context-overflow" + : undefined; +} + async function command(built: ReturnType, timeoutMs: number) { return await runBoundedCommand({ command: built.command, @@ -555,12 +571,13 @@ export async function executeCampaign(options: { const tokens = tokenDelta(before, after); if (!tokens.available) throw new Error("vLLM token counters were unavailable or reset"); const finalOutput = extractFinalAssistantOutput(run.agent, invocation.stdout); - const classifiedFailure = - run.phase !== "static-visibility" && - invocation.exit_code !== 0 && - isContextOverflow(invocation.stdout, invocation.stderr) - ? ("context-overflow" as const) - : undefined; + const classifiedFailure = classifyInvocationFailure({ + phase: run.phase, + exitCode: invocation.exit_code, + timedOut: invocation.timed_out, + stdout: invocation.stdout, + stderr: invocation.stderr, + }); const rawEvidence: SanitizedRunEvidence = { run_id: run.run_id, recorder_events: recorderEvents, diff --git a/test/bench/tool-disclosure-drivers.test.ts b/test/bench/tool-disclosure-drivers.test.ts index 541a048801..1f79c784b6 100644 --- a/test/bench/tool-disclosure-drivers.test.ts +++ b/test/bench/tool-disclosure-drivers.test.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; + import { describe, expect, it } from "vitest"; import { @@ -47,6 +49,22 @@ describe("tool-disclosure agent drivers", () => { ).toThrow("sandboxName contains unsupported characters"); }); + it.skipIf(process.platform === "win32")("emits POSIX-valid driver scripts", () => { + for (const agent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { + const command = buildAgentDriverCommand({ + agent, + sandboxName: "bench-syntax", + prompt: "Return BENCH_OK_123", + sessionId: "bench-session-syntax", + }); + const syntax = spawnSync("sh", ["-n"], { + encoding: "utf8", + input: command.args.at(-1), + }); + expect(syntax.status, syntax.stderr).toBe(0); + } + }); + it("extracts assistant replies without accepting tool-result fields", () => { expect( extractFinalAssistantOutput( diff --git a/test/bench/tool-disclosure-execute.test.ts b/test/bench/tool-disclosure-execute.test.ts index 27f02450ea..6975b9ba8c 100644 --- a/test/bench/tool-disclosure-execute.test.ts +++ b/test/bench/tool-disclosure-execute.test.ts @@ -11,6 +11,7 @@ import { assertSetupRetryAllowed, type AttemptJournalEntry, type CampaignAttestation, + classifyInvocationFailure, materializeAttemptJournal, recoverAttemptJournal, } from "../../scripts/bench/tool-disclosure/execute"; @@ -60,6 +61,27 @@ function attestationFixture(): { } describe("tool-disclosure attempt journal", () => { + it("gives timeout precedence over context-overflow text", () => { + expect( + classifyInvocationFailure({ + phase: "primary", + exitCode: null, + timedOut: true, + stdout: "", + stderr: "maximum context length exceeded", + }), + ).toBeUndefined(); + expect( + classifyInvocationFailure({ + phase: "primary", + exitCode: 1, + timedOut: false, + stdout: "", + stderr: "maximum context length exceeded", + }), + ).toBe("context-overflow"); + }); + it("retries only failures that occur before agent invocation", () => { expect(() => assertSetupRetryAllowed(false, "run-before-invocation")).not.toThrow(); expect(() => From daa4224ff48eb8645550d166e412d99be3c647d2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 15:15:22 -0700 Subject: [PATCH 05/31] fix(bench): enforce benchmark isolation boundaries Signed-off-by: Aaron Erickson --- .../progressive-tool-disclosure-benchmark.mdx | 2 + scripts/bench/tool-disclosure/drivers.ts | 7 ++- scripts/bench/tool-disclosure/execute.ts | 50 +++++++++++++---- .../bench/tool-disclosure/openclaw-fixture.ts | 13 ++++- scripts/bench/tool-disclosure/quick-tunnel.ts | 55 ++++++++++++++----- scripts/bench/tool-disclosure/recorder.ts | 2 + test/bench/tool-disclosure-drivers.test.ts | 13 +++++ test/bench/tool-disclosure-execute.test.ts | 34 ++++++++++++ test/bench/tool-disclosure-fixture.test.ts | 22 +++++++- test/bench/tool-disclosure-mcp-server.test.ts | 20 +++++-- .../tool-disclosure-quick-tunnel.test.ts | 29 ++++++++++ test/bench/tool-disclosure-recorder.test.ts | 18 ++++++ 12 files changed, 229 insertions(+), 36 deletions(-) diff --git a/docs/inference/progressive-tool-disclosure-benchmark.mdx b/docs/inference/progressive-tool-disclosure-benchmark.mdx index 456e2ed91e..b52625582e 100644 --- a/docs/inference/progressive-tool-disclosure-benchmark.mdx +++ b/docs/inference/progressive-tool-disclosure-benchmark.mdx @@ -130,6 +130,8 @@ Do not publish this file because it can contain local routing and sandbox identi Execution verifies the live vLLM container ID, Docker image ID, model and revision, parser names, public flags, and process start time. It also verifies each live sandbox status against its expected agent, disclosure mode, immutable image ID, and instance ID. Only hashes of private container identities and reviewed public configuration enter the evidence bundle. +Each campaign generates a new MCP bearer token and shares it only among that campaign's five short-lived catalog servers; the token is never written to the public evidence artifacts. +The quick-tunnel process disables config-file discovery and receives an isolated temporary `HOME` and `XDG_CONFIG_HOME`, removed when the tunnel closes, so it cannot load the operator's ambient Cloudflare account configuration or credentials. This attestation path expects vLLM and the DGX Station's OpenShell sandboxes to use the Docker driver so `docker inspect` can verify the live state. ```bash diff --git a/scripts/bench/tool-disclosure/drivers.ts b/scripts/bench/tool-disclosure/drivers.ts index 51548f1778..409b408085 100644 --- a/scripts/bench/tool-disclosure/drivers.ts +++ b/scripts/bench/tool-disclosure/drivers.ts @@ -52,11 +52,12 @@ function shellScriptFor(agent: ToolDisclosureAgent, prompt: string, sessionId: s "set -a", "[ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env", "set +a", - `payload=$(printf '%s' '${payload}' | base64 -d)`, + `payload='${payload}'`, 'if [ -n "${API_SERVER_KEY:-}" ]; then', - ' exec curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H \'Content-Type: application/json\' -H "Authorization: Bearer ${API_SERVER_KEY}" --data-binary "$payload"', + " printf '%s' \"$payload\" | base64 -d | curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer ${API_SERVER_KEY}\" --data-binary @-", + " exit $?", "fi", - "exec curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary \"$payload\"", + "printf '%s' \"$payload\" | base64 -d | curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary @-", ].join("\n"); } diff --git a/scripts/bench/tool-disclosure/execute.ts b/scripts/bench/tool-disclosure/execute.ts index e77b54a994..f2c818a56f 100644 --- a/scripts/bench/tool-disclosure/execute.ts +++ b/scripts/bench/tool-disclosure/execute.ts @@ -58,6 +58,22 @@ export function assertSetupRetryAllowed( } } +/** Continue setup retries across process restarts without resetting the allowance. */ +export function nextSetupAttempt( + entries: readonly AttemptJournalEntry[], + runId: string, + retrySetupFailures: number, +): number { + if (!Number.isSafeInteger(retrySetupFailures) || retrySetupFailures < 0) { + throw new Error("retry_setup_failures must be a non-negative integer"); + } + const used = entries.filter( + (entry) => entry.run.run_id === runId && entry.run.outcome === "setup-error", + ).length; + if (used > retrySetupFailures) throw new Error(`run ${runId} exhausted setup retries`); + return used; +} + export function recoverAttemptJournal(outputDir: string): AttemptJournalEntry[] { const file = artifactPath(outputDir, "attempt-journal.jsonl"); if (!fs.existsSync(file)) return []; @@ -141,6 +157,19 @@ function sha256(value: string): string { return createHash("sha256").update(value, "utf8").digest("hex"); } +/** Accept only the single inspect result whose immutable ID was requested. */ +export function selectInspectedContainer( + value: unknown, + expectedId: string, +): (T & { Id: string }) | undefined { + if (!Array.isArray(value) || value.length !== 1) return undefined; + return ( + (value as Array<(T & { Id: string }) | null>).find( + (candidate) => candidate?.Id === expectedId, + ) ?? undefined + ); +} + async function attestVllmContainer(options: { config: LiveCampaignConfiguration; manifest: ToolDisclosureManifest; @@ -158,12 +187,11 @@ async function attestVllmContainer(options: { if (result.exit_code !== 0 || result.timed_out || result.output_truncated) { throw new Error("vLLM container inspection failed"); } - const containers = JSON.parse(result.stdout) as Array<{ + const container = selectInspectedContainer<{ Id?: string; Image?: string; Config?: { Cmd?: string[]; Entrypoint?: string[]; Env?: string[] }; - }>; - const container = containers[0]; + }>(JSON.parse(result.stdout) as unknown, options.config.vllm_container_id); const expected = options.manifest.inference; const publicConfig = JSON.stringify({ cmd: container?.Config?.Cmd ?? [], @@ -178,9 +206,7 @@ async function attestVllmContainer(options: { ), }); if ( - containers.length !== 1 || !container || - container.Id !== options.config.vllm_container_id || container.Image !== expected.container_digest || ![ expected.model_id, @@ -235,16 +261,13 @@ async function attestSandbox(options: { if (inspected.exit_code !== 0 || inspected.timed_out || inspected.output_truncated) { throw new Error(`container inspection failed for ${options.cell}`); } - const containers = JSON.parse(inspected.stdout) as Array<{ + const container = selectInspectedContainer<{ Id?: string; Image?: string; Config?: { Env?: string[] }; - }>; - const container = containers[0]; + }>(JSON.parse(inspected.stdout) as unknown, instanceId); if ( - containers.length !== 1 || !container || - container.Id !== instanceId || container.Image !== imageDigest || !container.Config?.Env?.includes(`NEMOCLAW_TOOL_DISCLOSURE=${mode}`) ) { @@ -509,8 +532,13 @@ export async function executeCampaign(options: { const task = run.task_id ? tasks.get(run.task_id) : undefined; const sandboxName = options.config.sandbox_names[sandboxKey(run)]; const mcp = run.agent === "openclaw" ? undefined : servers.get(run.catalog_size); + const firstAttempt = nextSetupAttempt( + journal, + run.run_id, + options.manifest.protocol.retry_setup_failures, + ); for ( - let attempt = 0; + let attempt = firstAttempt; attempt <= options.manifest.protocol.retry_setup_failures; attempt += 1 ) { diff --git a/scripts/bench/tool-disclosure/openclaw-fixture.ts b/scripts/bench/tool-disclosure/openclaw-fixture.ts index a8a3c4ecd6..72d1d49f39 100644 --- a/scripts/bench/tool-disclosure/openclaw-fixture.ts +++ b/scripts/bench/tool-disclosure/openclaw-fixture.ts @@ -113,9 +113,20 @@ RUN mkdir -p /sandbox/.openclaw/extensions \\\n && cp -a /opt/nemoclaw-tool-disc WORKDIR /opt/nemoclaw\n`; } +/** Guard the renderer boundary so the validated base and emitted ARG cannot diverge. */ +export function assertDockerfileSandboxBase(content: string, expected: string): void { + assertImmutableSandboxBase(expected); + const [argument, from] = content.split("\n", 2); + if (argument !== `ARG SANDBOX_BASE=${expected}` || from !== "FROM ${SANDBOX_BASE}") { + throw new Error("generated Dockerfile sandbox base does not match"); + } +} + /** Generate a self-contained custom-image context for an OpenClaw catalog size. */ export function writeOpenClawFixture(options: OpenClawFixtureOptions): OpenClawFixtureResult { assertImmutableSandboxBase(options.sandboxBase); + const renderedDockerfile = dockerfile(options.sandboxBase); + assertDockerfileSandboxBase(renderedDockerfile, options.sandboxBase); const root = path.resolve(options.outputDir); if (fs.existsSync(root)) { const stat = fs.lstatSync(root); @@ -166,7 +177,7 @@ export function writeOpenClawFixture(options: OpenClawFixtureOptions): OpenClawF 2, )}\n`, ); - writeNewFile(root, "Dockerfile", dockerfile(options.sandboxBase)); + writeNewFile(root, "Dockerfile", renderedDockerfile); writeNewFile(root, ".dockerignore", "*\n!Dockerfile\n!plugin\n!plugin/**\n"); return { diff --git a/scripts/bench/tool-disclosure/quick-tunnel.ts b/scripts/bench/tool-disclosure/quick-tunnel.ts index ef4e140720..4d8836b932 100644 --- a/scripts/bench/tool-disclosure/quick-tunnel.ts +++ b/scripts/bench/tool-disclosure/quick-tunnel.ts @@ -2,10 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import { type ChildProcess, spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; -const ORIGIN_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com(?=$|[\s"'\\/])/iu; -// Deliberately omit HOME/XDG_CONFIG_HOME so cloudflared cannot load ambient -// account credentials or configuration into this ephemeral benchmark tunnel. +const ORIGIN_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com(?=$|[\s"'\\/])/giu; +// Ambient HOME/XDG_CONFIG_HOME are not allowlisted; the child receives an +// isolated replacement so it cannot load operator credentials or configuration. const SAFE_ENV = new Set([ "PATH", "TMPDIR", @@ -29,7 +32,7 @@ export interface QuickTunnel { } export function parseQuickTunnelOrigin(text: string): string | null { - return text.match(ORIGIN_PATTERN)?.[0] ?? null; + return [...text.matchAll(ORIGIN_PATTERN)].at(-1)?.[0] ?? null; } export function buildQuickTunnelArgs(port: number): string[] { @@ -38,6 +41,7 @@ export function buildQuickTunnelArgs(port: number): string[] { } return [ "tunnel", + "--config=", "--no-autoupdate", "--protocol", "http2", @@ -48,14 +52,18 @@ export function buildQuickTunnelArgs(port: number): string[] { ]; } -function subprocessEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { +export function buildQuickTunnelEnvironment( + env: NodeJS.ProcessEnv, + isolatedHome: string, +): NodeJS.ProcessEnv { + if (!path.isAbsolute(isolatedHome)) throw new Error("quick-tunnel home must be absolute"); const selected: NodeJS.ProcessEnv = {}; for (const [name, value] of Object.entries(env)) { if (value !== undefined && (SAFE_ENV.has(name) || name.startsWith("LC_"))) { selected[name] = value; } } - return selected; + return { ...selected, HOME: isolatedHome, XDG_CONFIG_HOME: isolatedHome }; } function stopChild(child: ChildProcess): Promise { @@ -103,11 +111,20 @@ export async function startQuickTunnel(options: { fetchImpl?: typeof fetch; env?: NodeJS.ProcessEnv; }): Promise { - const child = spawn(options.binary ?? "cloudflared", buildQuickTunnelArgs(options.port), { - detached: false, - env: subprocessEnv(options.env ?? process.env), - stdio: ["ignore", "pipe", "pipe"], - }); + const args = buildQuickTunnelArgs(options.port); + const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-")); + const removeIsolatedHome = (): void => fs.rmSync(isolatedHome, { recursive: true, force: true }); + let child: ChildProcess; + try { + child = spawn(options.binary ?? "cloudflared", args, { + detached: false, + env: buildQuickTunnelEnvironment(options.env ?? process.env, isolatedHome), + stdio: ["ignore", "pipe", "pipe"], + }); + } catch (error) { + removeIsolatedHome(); + throw error; + } const timeoutMs = options.timeoutMs ?? 45_000; const fetchImpl = options.fetchImpl ?? fetch; let origin: string | null = null; @@ -115,7 +132,7 @@ export async function startQuickTunnel(options: { let spawnError: Error | undefined; const inspect = (chunk: Buffer | string): void => { const candidate = `${carry}${chunk.toString()}`; - origin ??= parseQuickTunnelOrigin(candidate); + origin = parseQuickTunnelOrigin(candidate) ?? origin; carry = candidate.slice(-512); }; child.stdout?.on("data", inspect); @@ -133,12 +150,22 @@ export async function startQuickTunnel(options: { return { origin: published, mcpUrl: `${published}/mcp`, - close: () => stopChild(child), + close: async () => { + try { + await stopChild(child); + } finally { + removeIsolatedHome(); + } + }, }; } await new Promise((resolve) => setTimeout(resolve, 250)); } - await stopChild(child); + try { + await stopChild(child); + } finally { + removeIsolatedHome(); + } if (spawnError) throw new Error(`cloudflared quick tunnel failed to start (${spawnError.name})`); throw new Error("cloudflared quick tunnel did not become ready before the timeout"); } diff --git a/scripts/bench/tool-disclosure/recorder.ts b/scripts/bench/tool-disclosure/recorder.ts index 5713334340..db5437f3e4 100644 --- a/scripts/bench/tool-disclosure/recorder.ts +++ b/scripts/bench/tool-disclosure/recorder.ts @@ -182,6 +182,8 @@ function parseUpstreamBaseUrl(value: string): URL { throw new Error("upstreamBaseUrl must use HTTP or HTTPS"); } const hostname = url.hostname.replace(/^\[/u, "").replace(/\]$/u, "").toLowerCase(); + // URL canonicalizes equivalent IPv6 spellings to ::1. Keep the plaintext + // allowlist exact; IPv4-mapped and scoped IPv6 forms are intentionally rejected. const loopback = hostname === "localhost" || hostname === "::1" || diff --git a/test/bench/tool-disclosure-drivers.test.ts b/test/bench/tool-disclosure-drivers.test.ts index 1f79c784b6..9284f346df 100644 --- a/test/bench/tool-disclosure-drivers.test.ts +++ b/test/bench/tool-disclosure-drivers.test.ts @@ -65,6 +65,19 @@ describe("tool-disclosure agent drivers", () => { } }); + it("streams the decoded Hermes payload directly to curl", () => { + const command = buildAgentDriverCommand({ + agent: "hermes", + sandboxName: "bench-hermes-stream", + prompt: "Return BENCH_OK_123", + sessionId: "bench-session-hermes-stream", + }); + const script = command.args.at(-1) ?? ""; + expect(script).toContain("base64 -d | curl"); + expect(script).toContain("--data-binary @-"); + expect(script).not.toContain("payload=$(printf"); + }); + it("extracts assistant replies without accepting tool-result fields", () => { expect( extractFinalAssistantOutput( diff --git a/test/bench/tool-disclosure-execute.test.ts b/test/bench/tool-disclosure-execute.test.ts index 6975b9ba8c..38773ea0c6 100644 --- a/test/bench/tool-disclosure-execute.test.ts +++ b/test/bench/tool-disclosure-execute.test.ts @@ -13,7 +13,9 @@ import { type CampaignAttestation, classifyInvocationFailure, materializeAttemptJournal, + nextSetupAttempt, recoverAttemptJournal, + selectInspectedContainer, } from "../../scripts/bench/tool-disclosure/execute"; import { validateCampaignAttestations } from "../../scripts/bench/tool-disclosure/run"; import { @@ -89,6 +91,38 @@ describe("tool-disclosure attempt journal", () => { ).toThrow(/discard this campaign/u); }); + it("preserves exhausted setup attempts and refuses another resume", () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); + try { + const entry = { + raw: { run_id: "run-exhausted", failure_outcome: "setup-error" }, + run: { run_id: "run-exhausted", outcome: "setup-error" }, + } as unknown as AttemptJournalEntry; + const attempts = [entry, structuredClone(entry)]; + const serialized = `${attempts.map((attempt) => JSON.stringify(attempt)).join("\n")}\n`; + const file = path.join(output, "attempt-journal.jsonl"); + fs.writeFileSync(file, serialized); + + expect(nextSetupAttempt([entry], "run-exhausted", 1)).toBe(1); + const recovered = recoverAttemptJournal(output); + expect(recovered).toEqual(attempts); + expect(() => nextSetupAttempt(recovered, "run-exhausted", 1)).toThrow( + /exhausted setup retries/u, + ); + expect(fs.readFileSync(file, "utf8")).toBe(serialized); + } finally { + fs.rmSync(output, { recursive: true, force: true }); + } + }); + + it("rejects ambiguous Docker inspect output regardless of order", () => { + const expected = { Id: "expected", Image: "sha256:expected" }; + const other = { Id: "other", Image: "sha256:other" }; + expect(selectInspectedContainer([expected], expected.Id)).toEqual(expected); + expect(selectInspectedContainer([other, expected], expected.Id)).toBeUndefined(); + expect(selectInspectedContainer([expected, other], expected.Id)).toBeUndefined(); + }); + it("recovers one trailing partial append and deterministically materializes evidence", () => { const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); try { diff --git a/test/bench/tool-disclosure-fixture.test.ts b/test/bench/tool-disclosure-fixture.test.ts index ca6c3bbde6..b0769d1f75 100644 --- a/test/bench/tool-disclosure-fixture.test.ts +++ b/test/bench/tool-disclosure-fixture.test.ts @@ -16,7 +16,10 @@ import { sha256Hex, } from "../../scripts/bench/tool-disclosure/catalog"; import { gradeTaskRun } from "../../scripts/bench/tool-disclosure/grading"; -import { writeOpenClawFixture } from "../../scripts/bench/tool-disclosure/openclaw-fixture"; +import { + assertDockerfileSandboxBase, + writeOpenClawFixture, +} from "../../scripts/bench/tool-disclosure/openclaw-fixture"; import { buildToolDisclosureSchedule } from "../../scripts/bench/tool-disclosure/schedule"; import { buildPrimaryTasks } from "../../scripts/bench/tool-disclosure/tasks"; import { @@ -93,6 +96,23 @@ describe("OpenClaw benchmark fixture", () => { ).toThrow(/canonical registry\/repository reference/u); expect(fs.existsSync(output)).toBe(false); }); + + it("rejects a generated Dockerfile base mismatch", () => { + const expected = `registry.example/base@sha256:${"1".repeat(64)}`; + const substituted = `registry.example/base@sha256:${"2".repeat(64)}`; + expect(() => + assertDockerfileSandboxBase( + `ARG SANDBOX_BASE=${substituted}\nFROM \${SANDBOX_BASE}\n`, + expected, + ), + ).toThrow(/does not match/u); + expect(() => + assertDockerfileSandboxBase( + `ARG SANDBOX_BASE=${expected}\nFROM registry.example/other@sha256:${"3".repeat(64)}\n`, + expected, + ), + ).toThrow(/does not match/u); + }); }); describe("benchmark grading", () => { diff --git a/test/bench/tool-disclosure-mcp-server.test.ts b/test/bench/tool-disclosure-mcp-server.test.ts index fbf5fd5a83..58e1422365 100644 --- a/test/bench/tool-disclosure-mcp-server.test.ts +++ b/test/bench/tool-disclosure-mcp-server.test.ts @@ -34,12 +34,6 @@ describe("synthetic benchmark MCP server", () => { try { const unauthorized = await fetch(address.local_url, { method: "POST", body: "{}" }); expect(unauthorized.status).toBe(401); - const staleCampaignToken = await rpc( - address.local_url, - "token-from-another-campaign", - "tools/list", - ); - expect(staleCampaignToken.status).toBe(401); const initialized = await rpc(address.local_url, token, "initialize", { protocolVersion: "2025-03-26", @@ -76,4 +70,18 @@ describe("synthetic benchmark MCP server", () => { await server.stop(); } }); + + it("rejects a bearer token reused from another campaign", async () => { + const server = new SyntheticMcpServer( + generateSyntheticCatalog({ size: 16, seed: "mcp-token-rotation-test" }), + "current-campaign-token", + ); + const address = await server.start(); + try { + const response = await rpc(address.local_url, "previous-campaign-token", "tools/list"); + expect(response.status).toBe(401); + } finally { + await server.stop(); + } + }); }); diff --git a/test/bench/tool-disclosure-quick-tunnel.test.ts b/test/bench/tool-disclosure-quick-tunnel.test.ts index 5f62140946..7634f84e11 100644 --- a/test/bench/tool-disclosure-quick-tunnel.test.ts +++ b/test/bench/tool-disclosure-quick-tunnel.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { buildQuickTunnelArgs, + buildQuickTunnelEnvironment, parseQuickTunnelOrigin, } from "../../scripts/bench/tool-disclosure/quick-tunnel"; @@ -16,9 +17,37 @@ describe("tool-disclosure quick tunnel", () => { expect(parseQuickTunnelOrigin("https://example.com/secret")).toBeNull(); }); + it("uses the latest origin when cloudflared reports more than one", () => { + expect( + parseQuickTunnelOrigin( + "old https://stale-123.trycloudflare.com\nnew https://current-456.trycloudflare.com\n", + ), + ).toBe("https://current-456.trycloudflare.com"); + }); + + it("omits ambient Cloudflare account configuration from the subprocess", () => { + expect( + buildQuickTunnelEnvironment( + { + PATH: "/usr/bin", + HOME: "/home/operator", + XDG_CONFIG_HOME: "/home/operator/.config", + LC_ALL: "C.UTF-8", + }, + "/tmp/isolated-cloudflared-home", + ), + ).toEqual({ + PATH: "/usr/bin", + LC_ALL: "C.UTF-8", + HOME: "/tmp/isolated-cloudflared-home", + XDG_CONFIG_HOME: "/tmp/isolated-cloudflared-home", + }); + }); + it("builds a loopback origin command and rejects invalid ports", () => { expect(buildQuickTunnelArgs(31337)).toEqual([ "tunnel", + "--config=", "--no-autoupdate", "--protocol", "http2", diff --git a/test/bench/tool-disclosure-recorder.test.ts b/test/bench/tool-disclosure-recorder.test.ts index 3ee67ec716..0ad3a67029 100644 --- a/test/bench/tool-disclosure-recorder.test.ts +++ b/test/bench/tool-disclosure-recorder.test.ts @@ -99,6 +99,24 @@ describe("tool-disclosure recording proxy", () => { ).toThrow("plaintext upstreamBaseUrl is allowed only on loopback"); }); + it("canonicalizes the IPv6 loopback and rejects mapped or non-loopback variants", () => { + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://[0:0:0:0:0:0:0:1]:8000/v1", + }), + ).not.toThrow(); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://[::ffff:127.0.0.1]:8000/v1", + }), + ).toThrow("plaintext upstreamBaseUrl is allowed only on loopback"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://[::2]:8000/v1", + }), + ).toThrow("plaintext upstreamBaseUrl is allowed only on loopback"); + }); + it("accepts bounded public scheduled run IDs and rejects unsafe IDs", () => { const proxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl: "https://inference.example/v1", From dec0bdcd9bfcc268f71e81609e860eafa202cee5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 15:32:55 -0700 Subject: [PATCH 06/31] fix(bench): restrict vLLM traffic to loopback Signed-off-by: Aaron Erickson --- scripts/bench/tool-disclosure/recorder.ts | 4 +-- scripts/bench/tool-disclosure/telemetry.ts | 4 +-- test/bench/tool-disclosure-recorder.test.ts | 29 +++++++++++++++----- test/bench/tool-disclosure-telemetry.test.ts | 15 ++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/scripts/bench/tool-disclosure/recorder.ts b/scripts/bench/tool-disclosure/recorder.ts index db5437f3e4..c238532f6c 100644 --- a/scripts/bench/tool-disclosure/recorder.ts +++ b/scripts/bench/tool-disclosure/recorder.ts @@ -188,8 +188,8 @@ function parseUpstreamBaseUrl(value: string): URL { hostname === "localhost" || hostname === "::1" || (isIP(hostname) === 4 && hostname.startsWith("127.")); - if (url.protocol === "http:" && !loopback) { - throw new Error("plaintext upstreamBaseUrl is allowed only on loopback"); + if (!loopback) { + throw new Error("upstreamBaseUrl is allowed only on loopback"); } if (url.username || url.password) { throw new Error("upstreamBaseUrl must not contain credentials"); diff --git a/scripts/bench/tool-disclosure/telemetry.ts b/scripts/bench/tool-disclosure/telemetry.ts index 6f41b037ea..f1282edb13 100644 --- a/scripts/bench/tool-disclosure/telemetry.ts +++ b/scripts/bench/tool-disclosure/telemetry.ts @@ -39,8 +39,8 @@ export function validatedTelemetryUrl(raw: string, endpoint: string): URL { if (url.username || url.password) { throw new Error("vLLM telemetry URL must not contain credentials"); } - if (url.protocol === "http:" && !isLoopback(url.hostname)) { - throw new Error("plaintext vLLM telemetry is allowed only on loopback"); + if (!isLoopback(url.hostname)) { + throw new Error("vLLM telemetry is allowed only on loopback"); } url.username = ""; url.password = ""; diff --git a/test/bench/tool-disclosure-recorder.test.ts b/test/bench/tool-disclosure-recorder.test.ts index 0ad3a67029..0783632800 100644 --- a/test/bench/tool-disclosure-recorder.test.ts +++ b/test/bench/tool-disclosure-recorder.test.ts @@ -78,25 +78,40 @@ describe("tool-disclosure recording proxy", () => { it("fails closed for non-loopback listeners and credential-bearing upstream URLs", () => { expect(() => createToolDisclosureRecordingProxy({ - upstreamBaseUrl: "https://inference.example/v1", + upstreamBaseUrl: "https://localhost:8000/v1", listenHost: "0.0.0.0", }), ).toThrow("must be exactly 127.0.0.1"); expect(() => createToolDisclosureRecordingProxy({ - upstreamBaseUrl: "https://user:password@inference.example/v1", + upstreamBaseUrl: "https://user:password@localhost:8000/v1", }), ).toThrow("must not contain credentials"); expect(() => createToolDisclosureRecordingProxy({ - upstreamBaseUrl: "https://inference.example/v1?api_key=private-key", + upstreamBaseUrl: "https://localhost:8000/v1?api_key=private-key", }), ).toThrow("must not contain a query or fragment"); expect(() => createToolDisclosureRecordingProxy({ upstreamBaseUrl: "http://inference.example/v1", }), - ).toThrow("plaintext upstreamBaseUrl is allowed only on loopback"); + ).toThrow("upstreamBaseUrl is allowed only on loopback"); + }); + + it("rejects non-loopback HTTPS recorder upstreams for the local benchmark", () => { + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + }), + ).toThrow("upstreamBaseUrl is allowed only on loopback"); + for (const upstreamBaseUrl of [ + "http://localhost:8000/v1", + "https://127.0.0.1:8000/v1", + "https://[::1]:8000/v1", + ]) { + expect(() => createToolDisclosureRecordingProxy({ upstreamBaseUrl })).not.toThrow(); + } }); it("canonicalizes the IPv6 loopback and rejects mapped or non-loopback variants", () => { @@ -109,17 +124,17 @@ describe("tool-disclosure recording proxy", () => { createToolDisclosureRecordingProxy({ upstreamBaseUrl: "http://[::ffff:127.0.0.1]:8000/v1", }), - ).toThrow("plaintext upstreamBaseUrl is allowed only on loopback"); + ).toThrow("upstreamBaseUrl is allowed only on loopback"); expect(() => createToolDisclosureRecordingProxy({ upstreamBaseUrl: "http://[::2]:8000/v1", }), - ).toThrow("plaintext upstreamBaseUrl is allowed only on loopback"); + ).toThrow("upstreamBaseUrl is allowed only on loopback"); }); it("accepts bounded public scheduled run IDs and rejects unsafe IDs", () => { const proxy = createToolDisclosureRecordingProxy({ - upstreamBaseUrl: "https://inference.example/v1", + upstreamBaseUrl: "https://localhost:8000/v1", }); const scheduledId = "c1--primary--openclaw--progressive--n512--single-01--r1"; expect(proxy.beginRun(scheduledId)).toBe(scheduledId); diff --git a/test/bench/tool-disclosure-telemetry.test.ts b/test/bench/tool-disclosure-telemetry.test.ts index d048b3eba5..6b0a1b83e0 100644 --- a/test/bench/tool-disclosure-telemetry.test.ts +++ b/test/bench/tool-disclosure-telemetry.test.ts @@ -48,6 +48,21 @@ vllm:generation_tokens_total{model_name="bench",engine="1"} 2 ); }); + it("rejects non-loopback HTTPS telemetry and tokenizer URLs for the local benchmark", () => { + expect(() => validatedTelemetryUrl("https://inference.example/v1", "/metrics")).toThrow( + "vLLM telemetry is allowed only on loopback", + ); + expect(validatedTelemetryUrl("https://localhost:8000/v1", "/metrics").toString()).toBe( + "https://localhost:8000/metrics", + ); + expect(validatedTelemetryUrl("https://127.0.0.1:8000/v1", "/tokenize").toString()).toBe( + "https://127.0.0.1:8000/tokenize", + ); + expect(validatedTelemetryUrl("https://[::1]:8000/v1", "/metrics").toString()).toBe( + "https://[::1]:8000/metrics", + ); + }); + it("reads token count variants without exposing response bodies", async () => { const fetchImpl = (async () => new Response(JSON.stringify({ tokens: [1, 2, 3, 4] }), { From 252674f1f2d7337b9d441a3699ff1da0ef98af0c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 16:08:06 -0700 Subject: [PATCH 07/31] refactor(bench): make disclosure protocol hardware neutral Signed-off-by: Aaron Erickson --- .../progressive-tool-disclosure-benchmark.mdx | 82 +++++++++------- scripts/bench/README.md | 10 +- scripts/bench/tool-disclosure/report.ts | 14 ++- scripts/bench/tool-disclosure/run.ts | 66 +++++++------ scripts/bench/tool-disclosure/types.ts | 14 ++- scripts/bench/tool-disclosure/validation.ts | 39 ++++++-- test/bench/tool-disclosure-report.test.ts | 97 +++++++++++++++---- test/bench/tool-disclosure-validation.test.ts | 70 +++++++++++-- 8 files changed, 283 insertions(+), 109 deletions(-) diff --git a/docs/inference/progressive-tool-disclosure-benchmark.mdx b/docs/inference/progressive-tool-disclosure-benchmark.mdx index b52625582e..d94a5cd2bb 100644 --- a/docs/inference/progressive-tool-disclosure-benchmark.mdx +++ b/docs/inference/progressive-tool-disclosure-benchmark.mdx @@ -1,11 +1,11 @@ --- # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Validate Progressive Tool Disclosure" +title: "Benchmark Progressive Tool Disclosure" sidebar-title: "Progressive Tool Disclosure Benchmark" -description: "Run the reproducible two-campaign benchmark that validates progressive tool-disclosure claims across NemoClaw agent surfaces." -description-agent: "Explains how to prepare, run, and summarize the progressive tool-disclosure benchmark on a DGX Station. Use when validating tool-schema token, task-success, or latency claims for OpenClaw, Hermes, and LangChain Deep Agents Code." -keywords: "progressive tool disclosure benchmark, DGX Station benchmark, tool schema tokens, MCP benchmark" +description: "Run the reproducible two-campaign benchmark that measures progressive tool disclosure across NemoClaw agent surfaces." +description-agent: "Explains how to prepare, run, and summarize the hardware-neutral progressive tool-disclosure benchmark. Use when measuring tool-schema tokens, task success, or latency for OpenClaw, Hermes, and LangChain Deep Agents Code." +keywords: "progressive tool disclosure benchmark, hardware-neutral benchmark, tool schema tokens, MCP benchmark" position: 5 --- @@ -26,7 +26,7 @@ Use the generated files without adding, removing, or reordering runs. | --- | ---: | --- | --- | | `static-visibility` | `16`, `64`, `256`, `512`, and `2209` | One exact capture at each size. | Measure the initial serialized tool-schema bytes, tokens, and visible tool count. | | `small-control` | `64` | `24` tasks, one repetition. | Check behavior before context pressure becomes large. | -| `primary` | `512` | `24` tasks, five repetitions. | Supply the paired observations used by the public claim gates. | +| `primary` | `512` | `24` tasks, five repetitions. | Supply the paired observations used by the reported claim gates. | | `large-stress` | `2209` | `8` tasks, one repetition. | Exercise the largest generated catalog without widening the primary claim. | The schedule contains `942` runs in each campaign and `1884` runs in total. @@ -50,7 +50,7 @@ Preparation writes these inputs: - `catalog.json` contains the deterministic synthetic tool catalog. - `primary-tasks.json` and `stress-tasks.json` contain the frozen task definitions. - `schedule.json` contains both campaigns in execution order. -- `manifest.template.json` contains the public evidence schema fields that you must complete. +- `manifest.template.json` contains the shareable evidence schema fields that you must complete. - The generated OpenClaw fixture directories contain native tool plugins for the scheduled catalog sizes. Copy `manifest.template.json` to `manifest.json` and replace every placeholder before the first run. @@ -59,45 +59,51 @@ Copy `manifest.template.json` to `manifest.json` and replace every placeholder b cp /manifest.template.json /manifest.json ``` -Record the full NemoClaw system-under-test and harness commit SHAs, exact model ID and revision, vLLM image digest and version, public serving flags, parser names, OpenShell version, agent versions, GPU and CPU description, driver and CUDA versions, RAM, and the observed power state. -Keep credentials, endpoint URLs, hostnames, usernames, reservation identifiers, and local paths out of the manifest. +Record the full NemoClaw system-under-test and harness commit SHAs, exact model ID and revision, vLLM image digest and version, serving flags, parser names, OpenShell version, agent versions, accelerator and CPU description, accelerator driver and runtime versions, RAM, and the observed power or performance state. +For a CPU-only run, set `accelerator_count` to `0`, `accelerator_type` to `none`, and the remaining accelerator fields to `not-applicable`. +Keep credentials, endpoint URLs, hostnames, usernames, and local paths out of the manifest. -The benchmark commands do not reserve Colossus infrastructure, start vLLM, or create sandboxes. -Prepare those resources first; the `execute` command starts the loopback recorder and synthetic MCP fixtures, drives the frozen schedule, and writes sanitized raw evidence plus derived run records. +The benchmark commands do not provision hardware, start vLLM, or create sandboxes. +Prepare the runtime environment first; the `execute` command starts the loopback recorder and synthetic MCP fixtures, drives the frozen schedule, and writes sanitized raw evidence plus derived run records. -## Reserve and preflight a DGX Station +## Provision and preflight the benchmark host -Use the current internal Colossus runbook to reserve a DGX Station and obtain host access. -Colossus pools, commands, and access routes are site-specific, so this public protocol does not prescribe an internal identifier or command spelling. +In principle, the benchmark can run on any hardware with a compatible host environment that can serve the recorded model and complete the frozen schedule. +The required runtime consists of Node.js and a POSIX-compatible shell, an OpenAI-compatible vLLM endpoint with `/metrics` and `/tokenize`, Docker-backed OpenShell sandboxes, `cloudflared` with outbound network access, and working OpenClaw, Hermes, and LangChain Deep Agents Code runtimes. +Provisioning and access procedures are outside the benchmark protocol. -The following lines are placeholders, not runnable commands: - -```text - - -``` - -On the reserved station, run the public preflight commands before starting inference. +Run host-appropriate preflight checks before starting inference. +The following examples cover common Linux and macOS environments; run only the commands available on the benchmark host. ```bash -nvidia-smi -nvidia-smi -q +uname -a +uname -m +lscpu # Linux CPU inventory +sysctl -n machdep.cpu.brand_string # macOS CPU inventory +free -h # Linux memory inventory +vm_stat # macOS memory inventory docker version openshell --version nemoclaw --version git rev-parse HEAD ``` -Confirm that the station has the expected GPU count and model, enough free GPU memory for the frozen model, the required container image by digest, and no competing GPU workload. -Set and record the intended power state before each campaign. -Do not publish raw Colossus output or raw preflight logs because they can contain internal host or reservation identifiers. +Accelerator inspection is optional and platform-specific; use the hardware vendor's supported inspection utility when applicable. +Confirm that the host matches the recorded processor and accelerator configuration, has enough available memory for the frozen model, has the required container image by digest, and has no competing workload that could affect the measurements. +Set and record the intended power or performance state before each campaign. +Do not publish raw preflight logs because they can contain hostnames, usernames, or local paths. ## Route inference through the recorder -Run one vLLM server and one benchmark recording proxy on the DGX Station. +Run one vLLM server and one benchmark recording proxy on the benchmark host. Configure the NemoClaw managed inference route so every model request from every benchmark sandbox passes through the proxy before reaching vLLM. +After vLLM starts, verify its local health endpoint before beginning a campaign. + +```bash +curl --fail --silent --show-error http://127.0.0.1:/health +``` ```text OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes @@ -110,14 +116,14 @@ OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes /metrics and /tokenize ``` -The recorder must bind to `127.0.0.1` and must not be exposed on a station or corporate network interface. +The recorder must bind to `127.0.0.1` and must not be exposed on a non-loopback network interface. It forwards OpenAI-compatible `/v1` requests without following redirects and records content-free request metadata, including the canonical tool-schema byte count, schema hash, visible tool count, tool names, status, monotonic timing, and run sequence. It does not retain prompts, messages, headers, request bodies, response bodies, model identifiers, endpoints, or errors from upstream bodies. Use vLLM `/tokenize` to count initial serialized tool-schema tokens with the exact served model tokenizer. Read prompt and generation token counters from vLLM `/metrics` immediately before and after each scheduled run. Keep benchmark concurrency at `1` so those process-level counter deltas remain attributable to one run. -Use temperature `0`, the manifest's frozen parser configuration, and the same public vLLM flags in both modes and both campaigns. +Use temperature `0`, the manifest's frozen parser configuration, and the same recorded vLLM flags in both modes and both campaigns. ## Run two fresh campaigns @@ -132,7 +138,7 @@ It also verifies each live sandbox status against its expected agent, disclosure Only hashes of private container identities and reviewed public configuration enter the evidence bundle. Each campaign generates a new MCP bearer token and shares it only among that campaign's five short-lived catalog servers; the token is never written to the public evidence artifacts. The quick-tunnel process disables config-file discovery and receives an isolated temporary `HOME` and `XDG_CONFIG_HOME`, removed when the tunnel closes, so it cannot load the operator's ambient Cloudflare account configuration or credentials. -This attestation path expects vLLM and the DGX Station's OpenShell sandboxes to use the Docker driver so `docker inspect` can verify the live state. +This attestation path expects vLLM and the OpenShell sandboxes to use the Docker driver so `docker inspect` can verify the live state. ```bash npm run bench:tool-disclosure -- execute \ @@ -149,7 +155,7 @@ For campaign 1: 5. Stop vLLM, stop the recorder, and destroy the campaign sandboxes. For campaign 2, repeat the same five steps with another fresh vLLM process, recorder event stream, and sandbox set. -Use the same station class, system-under-test and harness revisions, model revision, image digest, parser configuration, and task artifacts. +Use the same hardware configuration, system-under-test and harness revisions, model revision, image digest, parser configuration, and task artifacts. If any of those controlled values changes, prepare a new benchmark ID and restart both campaigns. The agent adapters must use the generated deterministic handlers and verify expected tool names, order, argument hashes, call count, unnecessary calls, and the returned nonce. @@ -175,14 +181,17 @@ The command consumes `manifest.json` and `runs.jsonl` and writes: - `evidence.json` with the public artifact inventory and SHA-256 values. - `SHA256SUMS` with the checksum manifest. -Verify the bundle on the DGX Station before you move it. +Verify the bundle on the benchmark host before you move it. ```bash cd -sha256sum -c SHA256SUMS +sha256sum -c SHA256SUMS # GNU/Linux +shasum -a 256 -c SHA256SUMS # macOS ``` -Treat every output as untrusted until you review it for credentials, endpoint URLs, hostnames, usernames, reservation identifiers, local paths, raw prompts, tool arguments, and response text. +Run the command available on the benchmark host. + +Treat every output as untrusted until you review it for credentials, endpoint URLs, hostnames, usernames, local paths, raw prompts, tool arguments, and response text. Do not hand-edit `summary.json` or the validated claims in `report.md`. If a correction changes an input or run record, regenerate the summary and checksums. @@ -205,13 +214,14 @@ A blocked gate remains visible in `report.md`, and the report does not emit the ## Interpret the scope -The benchmark supports claims about the recorded commits, agents, model and serving configuration, DGX Station class, deterministic synthetic catalog, and task corpus. +The benchmark supports claims about the recorded commits, agents, model, serving configuration, hardware configuration, deterministic synthetic catalog, and task corpus. It does not establish a universal result for other models, hardware, tool distributions, prompts, concurrency levels, or production workloads. +Evaluate another hardware configuration with a separate complete two-campaign run and report that result with its own recorded environment metadata. Progressive disclosure is a context optimization, not an authorization boundary. The OpenClaw fixture measures its native catalog path and does not establish that mcporter-managed MCP schemas are compacted. The `2209`-tool stress phase supplies boundary evidence but does not widen claims beyond the `512`-tool primary phase. -Two fresh campaigns on one station class demonstrate repeatability, not cross-hardware generality. +Two fresh campaigns on one hardware configuration demonstrate repeatability for that configuration, not cross-hardware generality. ## Related topics diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 0ae6766f9a..6fa7f99ca1 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -126,6 +126,12 @@ visibility for OpenClaw, Hermes, and LangChain Deep Agents Code against one deterministic synthetic catalog and task corpus. It is a claim-validation harness, not an extension of the advisory value benchmark above. +In principle, it can run on any hardware with a compatible host environment +that supports the recorded model, Node.js and a POSIX-compatible shell, an +OpenAI-compatible vLLM endpoint with `/metrics` and `/tokenize`, Docker-backed +OpenShell sandboxes, `cloudflared` with outbound network access, and the three +agent runtimes. Results apply to the recorded hardware and configuration; +evaluating another hardware configuration requires a separate complete run. Prepare a new evidence directory with the frozen catalog, task sets, schedule, manifest template, and generated OpenClaw fixtures: @@ -169,5 +175,5 @@ It derives public claim text mechanically and leaves claims blocked unless the required confidence-interval gate passes for every agent in both campaigns. See the [progressive tool-disclosure benchmark protocol](../../docs/inference/progressive-tool-disclosure-benchmark.mdx) -for the DGX Station workflow, recorder topology, artifact rules, claim gates, -and limitations. +for the hardware-neutral workflow, recorder topology, artifact rules, claim +gates, and limitations. diff --git a/scripts/bench/tool-disclosure/report.ts b/scripts/bench/tool-disclosure/report.ts index 774e7fb0e9..4e5cca1b2c 100644 --- a/scripts/bench/tool-disclosure/report.ts +++ b/scripts/bench/tool-disclosure/report.ts @@ -59,10 +59,14 @@ export function renderToolDisclosureMarkdown( "", "| Field | Recorded value |", "| --- | --- |", - `| GPU | ${cell(`${manifest.environment.gpu_count} × ${manifest.environment.gpu_model}`)} |`, - `| GPU architecture | ${cell(manifest.environment.gpu_architecture)} |`, - `| Driver / CUDA | ${cell( - `${manifest.environment.gpu_driver_version} / ${manifest.environment.cuda_version}`, + `| Accelerator | ${cell( + manifest.environment.accelerator_count === 0 + ? "none" + : `${manifest.environment.accelerator_count} × ${manifest.environment.accelerator_type}: ${manifest.environment.accelerator_model}`, + )} |`, + `| Accelerator architecture | ${cell(manifest.environment.accelerator_architecture)} |`, + `| Accelerator driver / runtime | ${cell( + `${manifest.environment.accelerator_driver_version} / ${manifest.environment.accelerator_runtime}`, )} |`, `| CPU / RAM | ${cell( `${manifest.environment.cpu_count} × ${manifest.environment.cpu_model}; ${formatNumber( @@ -87,7 +91,7 @@ export function renderToolDisclosureMarkdown( .map((agent) => `${agent}=${manifest.environment.agent_versions[agent]}`) .join(", "), )} |`, - `| Power state | ${cell(manifest.environment.power_state)} |`, + `| Power / performance state | ${cell(manifest.environment.power_state)} |`, "", "## Exact initial schema visibility", "", diff --git a/scripts/bench/tool-disclosure/run.ts b/scripts/bench/tool-disclosure/run.ts index 49c9ee4805..d842392245 100644 --- a/scripts/bench/tool-disclosure/run.ts +++ b/scripts/bench/tool-disclosure/run.ts @@ -159,7 +159,10 @@ function parseArguments(args: readonly string[]): ParsedArguments { } function gitOutput(args: readonly string[]): string { - return execFileSync("git", args, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + return execFileSync("git", args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); } function revision(ref: string, worktreeClean: boolean): ToolDisclosureManifest["sut"] { @@ -209,7 +212,10 @@ function manifestTemplate(options: { modes: TOOL_DISCLOSURE_MODES, catalog_sizes: STATIC_CATALOG_SIZES, primary_catalog_size: 512, - tasks: options.tasks.map((task) => ({ task_id: task.id, kind: task.kind })), + tasks: options.tasks.map((task) => ({ + task_id: task.id, + kind: task.kind, + })), repetitions: { "small-control": 1, primary: 5, "large-stress": 1 }, execution_seed: options.executionSeed, bootstrap_samples: DEFAULT_BOOTSTRAP_SAMPLES, @@ -218,40 +224,44 @@ function manifestTemplate(options: { retry_setup_failures: 1, }, environment: { - operating_system: "RECORD_ON_DGX", - architecture: "RECORD_ON_DGX", - cpu_model: "RECORD_ON_DGX", + operating_system: "RECORD_BEFORE_EXECUTION", + architecture: "RECORD_BEFORE_EXECUTION", + cpu_model: "RECORD_BEFORE_EXECUTION", cpu_count: 0, ram_gib: 0, - gpu_model: "RECORD_ON_DGX", - gpu_architecture: "RECORD_ON_DGX", - gpu_count: 0, - gpu_driver_version: "RECORD_ON_DGX", - cuda_version: "RECORD_ON_DGX", - power_state: "RECORD_ON_DGX", - openshell_version: "RECORD_ON_DGX", + accelerator_type: "RECORD_BEFORE_EXECUTION", + accelerator_model: "RECORD_BEFORE_EXECUTION", + accelerator_architecture: "RECORD_BEFORE_EXECUTION", + accelerator_count: 0, + accelerator_driver_version: "RECORD_BEFORE_EXECUTION", + accelerator_runtime: "RECORD_BEFORE_EXECUTION", + power_state: "RECORD_BEFORE_EXECUTION", + openshell_version: "RECORD_BEFORE_EXECUTION", agent_versions: { - openclaw: "RECORD_ON_DGX", - hermes: "RECORD_ON_DGX", - "langchain-deepagents-code": "RECORD_ON_DGX", + openclaw: "RECORD_BEFORE_EXECUTION", + hermes: "RECORD_BEFORE_EXECUTION", + "langchain-deepagents-code": "RECORD_BEFORE_EXECUTION", }, sandbox_image_digests: Object.fromEntries( TOOL_DISCLOSURE_AGENTS.flatMap((agent) => TOOL_DISCLOSURE_MODES.flatMap((mode) => - STATIC_CATALOG_SIZES.map((size) => [`${agent}:${mode}:${size}`, "RECORD_ON_DGX"]), + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + "RECORD_BEFORE_EXECUTION", + ]), ), ), ), }, inference: { api: "chat-completions", - model_id: "RECORD_ON_DGX", - model_revision: "RECORD_ON_DGX", - container_image: "RECORD_ON_DGX", - container_digest: "RECORD_ON_DGX", - vllm_version: "RECORD_ON_DGX", - tool_call_parser: "RECORD_ON_DGX", - reasoning_parser: "RECORD_ON_DGX", + model_id: "RECORD_BEFORE_EXECUTION", + model_revision: "RECORD_BEFORE_EXECUTION", + container_image: "RECORD_BEFORE_EXECUTION", + container_digest: "RECORD_BEFORE_EXECUTION", + vllm_version: "RECORD_BEFORE_EXECUTION", + tool_call_parser: "RECORD_BEFORE_EXECUTION", + reasoning_parser: "RECORD_BEFORE_EXECUTION", temperature: 0, concurrency: 1, prefix_caching_enabled: false, @@ -388,7 +398,7 @@ function prepare(options: ParsedArguments): void { ); writeChecksumManifest(outputDir, PREPARED_FILES); process.stdout.write( - `Prepared ${schedule.length} runs in ${outputDir}. Copy manifest.template.json to manifest.json and replace every RECORD_ON_DGX value before execution.\n`, + `Prepared ${schedule.length} runs in ${outputDir}. Copy manifest.template.json to manifest.json and replace every RECORD_BEFORE_EXECUTION value before execution.\n`, ); } @@ -422,8 +432,8 @@ function evidenceArtifact( function assertManifestCompleted(manifest: ToolDisclosureManifest): void { const serialized = JSON.stringify(manifest); - if (serialized.includes("RECORD_ON_DGX")) { - throw new Error("manifest.json still contains RECORD_ON_DGX placeholders"); + if (serialized.includes("RECORD_BEFORE_EXECUTION")) { + throw new Error("manifest.json still contains RECORD_BEFORE_EXECUTION placeholders"); } } @@ -619,7 +629,9 @@ function summarize(options: ParsedArguments): void { writeTextArtifact( outputDir, "report.md", - renderToolDisclosureMarkdown(manifest, summary, { artifacts: reportInputs }), + renderToolDisclosureMarkdown(manifest, summary, { + artifacts: reportInputs, + }), ); const artifacts = [...reportInputs, evidenceArtifact(outputDir, "report.md", "report")]; const bundle: EvidenceBundle = { diff --git a/scripts/bench/tool-disclosure/types.ts b/scripts/bench/tool-disclosure/types.ts index 42a769155c..60a11a3578 100644 --- a/scripts/bench/tool-disclosure/types.ts +++ b/scripts/bench/tool-disclosure/types.ts @@ -14,6 +14,8 @@ export const TOOL_DISCLOSURE_SCHEMA_VERSION = "nemoclaw.tool_disclosure_bench.v1 export const DEFAULT_BOOTSTRAP_SAMPLES = 10_000; export const DEFAULT_BOOTSTRAP_SEED = 0x4e_56_44_41; export const DEFAULT_NONINFERIORITY_MARGIN_PP = -5; +export const NO_ACCELERATOR_TYPE = "none" as const; +export const NOT_APPLICABLE = "not-applicable" as const; export type ToolDisclosureSchemaVersion = typeof TOOL_DISCLOSURE_SCHEMA_VERSION; export type ToolDisclosureAgent = "langchain-deepagents-code" | "hermes" | "openclaw"; @@ -48,11 +50,13 @@ export interface BenchmarkEnvironment { cpu_model: string; cpu_count: number; ram_gib: number; - gpu_model: string; - gpu_architecture: string; - gpu_count: number; - gpu_driver_version: string; - cuda_version: string; + /** Generic accelerator metadata. Use count 0, type "none", and explicit not-applicable values for CPU-only runs. */ + accelerator_type: string; + accelerator_model: string; + accelerator_architecture: string; + accelerator_count: number; + accelerator_driver_version: string; + accelerator_runtime: string; power_state: string; openshell_version: string; agent_versions: Record; diff --git a/scripts/bench/tool-disclosure/validation.ts b/scripts/bench/tool-disclosure/validation.ts index 61354e61ed..86d61f8301 100644 --- a/scripts/bench/tool-disclosure/validation.ts +++ b/scripts/bench/tool-disclosure/validation.ts @@ -17,6 +17,8 @@ import { DEFAULT_BOOTSTRAP_SAMPLES, DEFAULT_BOOTSTRAP_SEED, DEFAULT_NONINFERIORITY_MARGIN_PP, + NO_ACCELERATOR_TYPE, + NOT_APPLICABLE, } from "./types"; function sameJson(left: unknown, right: unknown): boolean { @@ -28,10 +30,11 @@ export function validateFrozenManifest(manifest: ToolDisclosureManifest): void { manifest.environment.operating_system, manifest.environment.architecture, manifest.environment.cpu_model, - manifest.environment.gpu_model, - manifest.environment.gpu_architecture, - manifest.environment.gpu_driver_version, - manifest.environment.cuda_version, + manifest.environment.accelerator_type, + manifest.environment.accelerator_model, + manifest.environment.accelerator_architecture, + manifest.environment.accelerator_driver_version, + manifest.environment.accelerator_runtime, manifest.environment.power_state, manifest.environment.openshell_version, ...TOOL_DISCLOSURE_AGENTS.map((agent) => manifest.environment.agent_versions[agent]), @@ -58,8 +61,8 @@ export function validateFrozenManifest(manifest: ToolDisclosureManifest): void { manifest.environment.cpu_count <= 0 || !Number.isFinite(manifest.environment.ram_gib) || manifest.environment.ram_gib <= 0 || - !Number.isSafeInteger(manifest.environment.gpu_count) || - manifest.environment.gpu_count <= 0 || + !Number.isSafeInteger(manifest.environment.accelerator_count) || + manifest.environment.accelerator_count < 0 || !/^sha256:[a-f0-9]{64}$/u.test(manifest.inference.container_digest) || manifest.inference.api !== "chat-completions" || manifest.inference.public_vllm_flags.some( @@ -67,10 +70,27 @@ export function validateFrozenManifest(manifest: ToolDisclosureManifest): void { ) || new Set(manifest.inference.public_vllm_flags).size !== manifest.inference.public_vllm_flags.length || - JSON.stringify(manifest).includes("RECORD_ON_DGX") + JSON.stringify(manifest).includes("RECORD_BEFORE_EXECUTION") ) { throw new Error("manifest is missing immutable claim-grade environment metadata"); } + const acceleratorMetadata = [ + manifest.environment.accelerator_model, + manifest.environment.accelerator_architecture, + manifest.environment.accelerator_driver_version, + manifest.environment.accelerator_runtime, + ]; + if ( + (manifest.environment.accelerator_count === 0 && + (manifest.environment.accelerator_type !== NO_ACCELERATOR_TYPE || + acceleratorMetadata.some((value) => value !== NOT_APPLICABLE))) || + (manifest.environment.accelerator_count > 0 && + (manifest.environment.accelerator_type === NO_ACCELERATOR_TYPE || + manifest.environment.accelerator_type === NOT_APPLICABLE || + acceleratorMetadata.some((value) => value === NOT_APPLICABLE))) + ) { + throw new Error("manifest accelerator metadata is inconsistent with accelerator count"); + } if ( !sameJson(manifest.protocol.agents, TOOL_DISCLOSURE_AGENTS) || !sameJson(manifest.protocol.modes, TOOL_DISCLOSURE_MODES) || @@ -268,7 +288,10 @@ export function validateCompleteEvidence(options: { throw new Error("schedule.json does not match the deterministic frozen schedule"); } const allTasks = [...primaryTasks.tasks, ...stressTasks.tasks]; - const expectedProtocolTasks = allTasks.map((task) => ({ task_id: task.id, kind: task.kind })); + const expectedProtocolTasks = allTasks.map((task) => ({ + task_id: task.id, + kind: task.kind, + })); if (!sameJson(manifest.protocol.tasks, expectedProtocolTasks)) { throw new Error("manifest tasks do not match the frozen task artifacts"); } diff --git a/test/bench/tool-disclosure-report.test.ts b/test/bench/tool-disclosure-report.test.ts index ddb7c4eec7..bc9aa0e837 100644 --- a/test/bench/tool-disclosure-report.test.ts +++ b/test/bench/tool-disclosure-report.test.ts @@ -29,10 +29,14 @@ const TASK_IDS = Array.from({ length: 24 }, (_, index) => `task-${index + 1}`); function makeManifest(overrides: Partial = {}): ToolDisclosureManifest { return { schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: "tool-disclosure-public-proof", + benchmark_id: "tool-disclosure-example", created_at: "2026-07-06T12:00:00.000Z", sut: { git_sha: "3a05b54e", git_ref: "v0.0.74", worktree_clean: true }, - harness: { git_sha: "1234abcd", git_ref: "benchmark-harness", worktree_clean: true }, + harness: { + git_sha: "1234abcd", + git_ref: "benchmark-harness", + worktree_clean: true, + }, campaigns: [ { campaign_id: "campaign-1", @@ -52,7 +56,10 @@ function makeManifest(overrides: Partial = {}): ToolDisc modes: ["direct", "progressive"], catalog_sizes: [512], primary_catalog_size: 512, - tasks: TASK_IDS.map((taskId) => ({ task_id: taskId, kind: "single-tool" as const })), + tasks: TASK_IDS.map((taskId) => ({ + task_id: taskId, + kind: "single-tool" as const, + })), repetitions: { "small-control": 1, primary: 1, "large-stress": 1 }, execution_seed: 7, bootstrap_samples: 500, @@ -63,14 +70,15 @@ function makeManifest(overrides: Partial = {}): ToolDisc environment: { operating_system: "Ubuntu 24.04", architecture: "x86_64", - cpu_model: "DGX Station CPU", + cpu_model: "Example CPU", cpu_count: 20, ram_gib: 128, - gpu_model: "NVIDIA DGX Station GPU", - gpu_architecture: "Blackwell", - gpu_count: 1, - gpu_driver_version: "999.1", - cuda_version: "13.0", + accelerator_type: "example-accelerator", + accelerator_model: "Example Accelerator", + accelerator_architecture: "example-architecture", + accelerator_count: 1, + accelerator_driver_version: "999.1", + accelerator_runtime: "example-runtime-1.0", power_state: "maximum-performance", openshell_version: "0.1.0", agent_versions: { @@ -83,7 +91,7 @@ function makeManifest(overrides: Partial = {}): ToolDisc api: "chat-completions", model_id: "deepseek-ai/DeepSeek-V4-Flash", model_revision: "revision-1", - container_image: "nvcr.io/nvidia/vllm:26.05.post1-py3", + container_image: "registry.example/vllm:1.0.0", container_digest: "sha256:container", vllm_version: "0.1.0", tool_call_parser: "deepseek_v3", @@ -124,7 +132,7 @@ function makeRun(options: RunOptions = {}): ToolDisclosureRun { runSequence += 1; return { schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: "tool-disclosure-public-proof", + benchmark_id: "tool-disclosure-example", campaign_id: options.campaign ?? "campaign-1", run_id: `run-${runSequence}`, phase, @@ -239,10 +247,30 @@ describe("tool-disclosure paired statistics", () => { it("averages repetitions within a task before resampling task pairs", () => { const runs = [ - makeRun({ mode: "direct", taskId: "task-a", repetition: 1, success: true }), - makeRun({ mode: "direct", taskId: "task-a", repetition: 2, success: false }), - makeRun({ mode: "progressive", taskId: "task-a", repetition: 1, success: true }), - makeRun({ mode: "progressive", taskId: "task-a", repetition: 2, success: true }), + makeRun({ + mode: "direct", + taskId: "task-a", + repetition: 1, + success: true, + }), + makeRun({ + mode: "direct", + taskId: "task-a", + repetition: 2, + success: false, + }), + makeRun({ + mode: "progressive", + taskId: "task-a", + repetition: 1, + success: true, + }), + makeRun({ + mode: "progressive", + taskId: "task-a", + repetition: 2, + success: true, + }), makeRun({ mode: "direct", taskId: "task-b", success: true }), makeRun({ mode: "progressive", taskId: "task-b", success: true }), ]; @@ -258,9 +286,21 @@ describe("tool-disclosure paired statistics", () => { describe("tool-disclosure visibility and claim gates", () => { it("rejects nondeterministic static visibility instead of rounding it", () => { const runs = [ - makeRun({ phase: "static-visibility", mode: "direct", schemaTokens: 1_000 }), - makeRun({ phase: "static-visibility", mode: "direct", schemaTokens: 1_001 }), - makeRun({ phase: "static-visibility", mode: "progressive", schemaTokens: 10 }), + makeRun({ + phase: "static-visibility", + mode: "direct", + schemaTokens: 1_000, + }), + makeRun({ + phase: "static-visibility", + mode: "direct", + schemaTokens: 1_001, + }), + makeRun({ + phase: "static-visibility", + mode: "progressive", + schemaTokens: 10, + }), ]; expect(() => summarizeStaticVisibility(runs)).toThrow("nondeterministic"); @@ -387,10 +427,31 @@ describe("tool-disclosure Markdown report", () => { expect(markdown).toContain("at least 99%"); expect(markdown).toContain("initial serialized tool-schema tokens"); expect(markdown).toContain("not an authorization boundary"); + expect(markdown).toContain("| Accelerator | 1 × example-accelerator: Example Accelerator |"); + expect(markdown).toContain("| Accelerator driver / runtime | 999.1 / example-runtime-1.0 |"); expect(markdown).toContain("summary.json"); expect(markdown).not.toContain("RAW-PROMPT-SECRET"); }); + it("renders CPU-only hardware without implying an accelerator", () => { + const manifest = makeManifest(); + manifest.environment.accelerator_type = "none"; + manifest.environment.accelerator_model = "not-applicable"; + manifest.environment.accelerator_architecture = "not-applicable"; + manifest.environment.accelerator_count = 0; + manifest.environment.accelerator_driver_version = "not-applicable"; + manifest.environment.accelerator_runtime = "not-applicable"; + const summary = buildToolDisclosureSummary(manifest, completeEvidence(), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + + const markdown = renderToolDisclosureMarkdown(manifest, summary); + + expect(markdown).toContain("| Accelerator | none |"); + expect(markdown).toContain("| Accelerator architecture | not-applicable |"); + expect(markdown).not.toContain("GPU"); + }); + it("rejects artifact paths so host locations cannot enter the report", () => { const manifest = makeManifest(); const summary = buildToolDisclosureSummary(manifest, completeEvidence(), { diff --git a/test/bench/tool-disclosure-validation.test.ts b/test/bench/tool-disclosure-validation.test.ts index a296f25f40..dc42408759 100644 --- a/test/bench/tool-disclosure-validation.test.ts +++ b/test/bench/tool-disclosure-validation.test.ts @@ -68,8 +68,16 @@ function buildManifest( schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, benchmark_id: "tool-disclosure-complete-evidence-fixture", created_at: "2026-07-06T12:00:00.000Z", - sut: { git_sha: "1".repeat(40), git_ref: "fixture-sut", worktree_clean: true }, - harness: { git_sha: "2".repeat(40), git_ref: "fixture-harness", worktree_clean: true }, + sut: { + git_sha: "1".repeat(40), + git_ref: "fixture-sut", + worktree_clean: true, + }, + harness: { + git_sha: "2".repeat(40), + git_ref: "fixture-harness", + worktree_clean: true, + }, campaigns: [ { campaign_id: "campaign-1", @@ -103,11 +111,12 @@ function buildManifest( cpu_model: "fixture-cpu", cpu_count: 1, ram_gib: 1, - gpu_model: "fixture-gpu", - gpu_architecture: "fixture-architecture", - gpu_count: 1, - gpu_driver_version: "fixture-driver", - cuda_version: "fixture-cuda", + accelerator_type: "fixture-accelerator-type", + accelerator_model: "fixture-accelerator", + accelerator_architecture: "fixture-architecture", + accelerator_count: 1, + accelerator_driver_version: "fixture-driver", + accelerator_runtime: "fixture-runtime", power_state: "fixture-power-state", openshell_version: "fixture-openshell", agent_versions: { @@ -236,7 +245,15 @@ function buildCompleteEvidenceFixture(): CompleteEvidenceFixture { ); } - return { catalog, manifest, primaryTasks, rawEvidence, runs, schedule, stressTasks }; + return { + catalog, + manifest, + primaryTasks, + rawEvidence, + runs, + schedule, + stressTasks, + }; } describe("tool-disclosure complete-evidence validation", () => { @@ -258,6 +275,43 @@ describe("tool-disclosure complete-evidence validation", () => { expect(() => validateFrozenManifest(mutableInferenceImage)).toThrow(/claim-grade/u); }); + it("accepts CPU-only hardware and requires explicit not-applicable accelerator metadata", () => { + const cpuOnly = structuredClone(fixture.manifest); + cpuOnly.environment.accelerator_type = "none"; + cpuOnly.environment.accelerator_model = "not-applicable"; + cpuOnly.environment.accelerator_architecture = "not-applicable"; + cpuOnly.environment.accelerator_count = 0; + cpuOnly.environment.accelerator_driver_version = "not-applicable"; + cpuOnly.environment.accelerator_runtime = "not-applicable"; + expect(() => validateFrozenManifest(cpuOnly)).not.toThrow(); + + const incompleteCpuOnly = structuredClone(cpuOnly); + incompleteCpuOnly.environment.accelerator_runtime = ""; + expect(() => validateFrozenManifest(incompleteCpuOnly)).toThrow(/claim-grade/u); + + const contradictoryCpuOnly = structuredClone(cpuOnly); + contradictoryCpuOnly.environment.accelerator_model = "fixture-accelerator"; + expect(() => validateFrozenManifest(contradictoryCpuOnly)).toThrow(/inconsistent/u); + }); + + it("accepts arbitrary accelerator metadata and rejects not-applicable positive counts", () => { + const arbitraryAccelerator = structuredClone(fixture.manifest); + arbitraryAccelerator.environment.accelerator_type = "tpu"; + arbitraryAccelerator.environment.accelerator_model = "fixture-tpu"; + arbitraryAccelerator.environment.accelerator_architecture = "fixture-tpu-architecture"; + arbitraryAccelerator.environment.accelerator_count = 4; + arbitraryAccelerator.environment.accelerator_driver_version = "fixture-tpu-driver"; + arbitraryAccelerator.environment.accelerator_runtime = "fixture-tpu-runtime"; + expect(() => validateFrozenManifest(arbitraryAccelerator)).not.toThrow(); + + arbitraryAccelerator.environment.accelerator_runtime = "not-applicable"; + expect(() => validateFrozenManifest(arbitraryAccelerator)).toThrow(/inconsistent/u); + + arbitraryAccelerator.environment.accelerator_runtime = "fixture-tpu-runtime"; + arbitraryAccelerator.environment.accelerator_type = "not-applicable"; + expect(() => validateFrozenManifest(arbitraryAccelerator)).toThrow(/inconsistent/u); + }); + it("accepts the exact 1,884-run frozen matrix and rejects reordered or tampered evidence", { timeout: 30_000, }, () => { From c684cf3d0ed3fd06b44e3d2a49d968035bcc8c2a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 16:29:03 -0700 Subject: [PATCH 08/31] fix(bench): harden benchmark evidence boundaries Signed-off-by: Aaron Erickson --- scripts/bench/tool-disclosure/execute.ts | 231 +++++++++++++++--- scripts/bench/tool-disclosure/recorder.ts | 12 +- scripts/bench/tool-disclosure/report.ts | 2 + scripts/bench/tool-disclosure/validation.ts | 69 ++++++ test/bench/tool-disclosure-execute.test.ts | 61 ++++- test/bench/tool-disclosure-recorder.test.ts | 19 +- test/bench/tool-disclosure-report.test.ts | 10 + test/bench/tool-disclosure-validation.test.ts | 45 ++++ 8 files changed, 408 insertions(+), 41 deletions(-) diff --git a/scripts/bench/tool-disclosure/execute.ts b/scripts/bench/tool-disclosure/execute.ts index f2c818a56f..7367117e31 100644 --- a/scripts/bench/tool-disclosure/execute.ts +++ b/scripts/bench/tool-disclosure/execute.ts @@ -25,7 +25,7 @@ import { readVllmTokenSnapshot, tokenDelta, } from "./telemetry"; -import type { ToolDisclosureManifest, ToolDisclosureRun } from "./types"; +import type { InferenceConfiguration, ToolDisclosureManifest, ToolDisclosureRun } from "./types"; export interface SanitizedRunEvidence { run_id: string; @@ -170,6 +170,199 @@ export function selectInspectedContainer( ); } +const PUBLIC_VLLM_ENV_NAMES = new Set([ + "VLLM_ATTENTION_BACKEND", + "VLLM_USE_V1", + "VLLM_WORKER_MULTIPROC_METHOD", +]); +const SHELL_EXECUTABLE_NAMES = new Set(["ash", "bash", "dash", "ksh", "sh", "zsh"]); +const SHELL_CONTROL_TOKENS = new Set(["&", "&&", ";", "|", "||"]); + +interface InspectedVllmContainer { + Id?: string; + Image?: string; + Config?: { + Cmd?: unknown; + Entrypoint?: unknown; + Env?: unknown; + Labels?: unknown; + }; +} + +export interface AttestedVllmConfiguration { + cmd: readonly string[]; + entrypoint: readonly string[]; + env: readonly string[]; +} + +function inspectedStringArray(value: unknown): string[] | undefined { + if (value === undefined || value === null) return []; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) return undefined; + return value; +} + +/** Tokenize the literal shell command used by Docker's `sh -c` form without expanding it. */ +function tokenizeShellCommand(command: string): string[] | undefined { + const tokens: string[] = []; + let word = ""; + let wordStarted = false; + let quote: "single" | "double" | undefined; + const pushWord = (): void => { + if (!wordStarted) return; + tokens.push(word); + word = ""; + wordStarted = false; + }; + + for (let index = 0; index < command.length; index += 1) { + const character = command[index]; + if (quote === "single") { + if (character === "'") quote = undefined; + else word += character; + continue; + } + if (quote === "double") { + if (character === '"') { + quote = undefined; + } else if (character === "\\") { + index += 1; + if (index >= command.length) return undefined; + word += command[index]; + } else { + word += character; + } + continue; + } + if (/\s/u.test(character)) { + pushWord(); + if (character === "\n") tokens.push(";"); + } else if (character === "'" || character === '"') { + quote = character === "'" ? "single" : "double"; + wordStarted = true; + } else if (character === "\\") { + index += 1; + if (index >= command.length) return undefined; + word += command[index]; + wordStarted = true; + } else if (character === ";" || character === "&" || character === "|") { + pushWord(); + const doubled = command[index + 1] === character; + tokens.push(doubled ? `${character}${character}` : character); + if (doubled) index += 1; + } else { + word += character; + wordStarted = true; + } + } + if (quote) return undefined; + pushWord(); + return tokens; +} + +function inspectedProcessTokens( + entrypoint: readonly string[], + cmd: readonly string[], +): string[] | undefined { + const argv = [...entrypoint, ...cmd]; + const shellIndex = argv.findIndex((value) => + SHELL_EXECUTABLE_NAMES.has(value.split("/").pop() ?? ""), + ); + if (shellIndex < 0) return argv; + const shellOption = argv[shellIndex + 1]; + if (!shellOption?.startsWith("-") || !shellOption.slice(1).includes("c")) return argv; + const commandIndex = shellIndex + 2; + const shellTokens = tokenizeShellCommand(argv[commandIndex] ?? ""); + if (!shellTokens) return undefined; + return [...argv.slice(0, commandIndex), ...shellTokens, ...argv.slice(commandIndex + 1)]; +} + +function optionMatches(args: readonly string[], name: string, value: string): boolean { + return args.some( + (argument, index) => + argument === `${name}=${value}` || (argument === name && args[index + 1] === value), + ); +} + +function containsSequence(values: readonly string[], expected: readonly string[]): boolean { + if (expected.length === 0) return false; + return values.some((_, index) => + expected.every((value, offset) => values[index + offset] === value), + ); +} + +function vllmInvocation( + tokens: readonly string[], +): { kind: "serve" | "api-server"; args: readonly string[] } | undefined { + const candidates: { kind: "serve" | "api-server"; start: number }[] = []; + for (let index = 0; index < tokens.length; index += 1) { + const executable = tokens[index].split("/").pop() ?? ""; + if (executable === "vllm" && tokens[index + 1] === "serve") { + candidates.push({ kind: "serve", start: index + 2 }); + } + if ( + /^python(?:\d+(?:\.\d+)*)?$/u.test(executable) && + tokens[index + 1] === "-m" && + tokens[index + 2] === "vllm.entrypoints.openai.api_server" + ) { + candidates.push({ kind: "api-server", start: index + 3 }); + } + } + if (candidates.length !== 1) return undefined; + const candidate = candidates[0]; + const boundary = tokens.findIndex( + (token, index) => index >= candidate.start && SHELL_CONTROL_TOKENS.has(token), + ); + return { + kind: candidate.kind, + args: tokens.slice(candidate.start, boundary < 0 ? undefined : boundary), + }; +} + +/** Extract only a structurally verified process configuration from Docker inspect. */ +export function readAttestedVllmConfiguration( + container: InspectedVllmContainer, + expected: InferenceConfiguration, +): AttestedVllmConfiguration | undefined { + const cmd = inspectedStringArray(container.Config?.Cmd); + const entrypoint = inspectedStringArray(container.Config?.Entrypoint); + const rawEnv = inspectedStringArray(container.Config?.Env); + if (!cmd || !entrypoint || !rawEnv) return undefined; + const tokens = inspectedProcessTokens(entrypoint, cmd); + if (!tokens) return undefined; + const invocation = vllmInvocation(tokens); + if (!invocation) return undefined; + const modelMatches = + (invocation.kind === "serve" && invocation.args[0] === expected.model_id) || + optionMatches(invocation.args, "--model", expected.model_id); + const publicFlagsMatch = expected.public_vllm_flags.every((flag) => { + const flagTokens = tokenizeShellCommand(flag); + return ( + flagTokens !== undefined && + flagTokens.every((token) => !SHELL_CONTROL_TOKENS.has(token)) && + containsSequence(invocation.args, flagTokens) + ); + }); + if ( + !modelMatches || + !optionMatches(invocation.args, "--revision", expected.model_revision) || + !optionMatches(invocation.args, "--tool-call-parser", expected.tool_call_parser) || + !optionMatches(invocation.args, "--reasoning-parser", expected.reasoning_parser) || + !publicFlagsMatch + ) { + return undefined; + } + + const publicEnv = rawEnv + .filter((entry) => PUBLIC_VLLM_ENV_NAMES.has(entry.slice(0, entry.indexOf("=")))) + .sort(); + if ( + new Set(publicEnv.map((entry) => entry.slice(0, entry.indexOf("=")))).size !== publicEnv.length + ) { + return undefined; + } + return { cmd, entrypoint, env: publicEnv }; +} + async function attestVllmContainer(options: { config: LiveCampaignConfiguration; manifest: ToolDisclosureManifest; @@ -187,40 +380,18 @@ async function attestVllmContainer(options: { if (result.exit_code !== 0 || result.timed_out || result.output_truncated) { throw new Error("vLLM container inspection failed"); } - const container = selectInspectedContainer<{ - Id?: string; - Image?: string; - Config?: { Cmd?: string[]; Entrypoint?: string[]; Env?: string[] }; - }>(JSON.parse(result.stdout) as unknown, options.config.vllm_container_id); + const container = selectInspectedContainer( + JSON.parse(result.stdout) as unknown, + options.config.vllm_container_id, + ); const expected = options.manifest.inference; - const publicConfig = JSON.stringify({ - cmd: container?.Config?.Cmd ?? [], - entrypoint: container?.Config?.Entrypoint ?? [], - env: (container?.Config?.Env ?? []).filter((value) => - [ - expected.model_id, - expected.model_revision, - expected.tool_call_parser, - expected.reasoning_parser, - ].some((needle) => value.includes(needle)), - ), - }); - if ( - !container || - container.Image !== expected.container_digest || - ![ - expected.model_id, - expected.model_revision, - expected.tool_call_parser, - expected.reasoning_parser, - ...expected.public_vllm_flags, - ].every((value) => result.stdout.includes(value)) - ) { + const publicConfig = container ? readAttestedVllmConfiguration(container, expected) : undefined; + if (!container || container.Image !== expected.container_digest || !publicConfig) { throw new Error("live vLLM container does not match the frozen manifest"); } return { inference_container_id_sha256: sha256(container.Id), - inference_config_sha256: sha256(publicConfig), + inference_config_sha256: sha256(JSON.stringify(publicConfig)), inference_image_digest: container.Image, }; } diff --git a/scripts/bench/tool-disclosure/recorder.ts b/scripts/bench/tool-disclosure/recorder.ts index c238532f6c..93bafbe1bd 100644 --- a/scripts/bench/tool-disclosure/recorder.ts +++ b/scripts/bench/tool-disclosure/recorder.ts @@ -755,11 +755,19 @@ export class ToolDisclosureRecordingProxy { { method: incomingRequest.method, headers: forwardedRequestHeaders(incomingRequest.headers, body.length), - // Native request does not follow redirects. A 3xx is passed to the - // benchmark client with its body and Location semantics intact. }, (upstreamResponse) => { statusCode = upstreamResponse.statusCode ?? 502; + if (statusCode >= 300 && statusCode < 400) { + upstreamResponse.resume(); + fixedResponse(response, 502, "upstream redirect rejected"); + finish({ + statusCode: 502, + outcome: "request-rejected", + errorReason: "proxy-failure", + }); + return; + } if (!response.headersSent && !response.destroyed) { response.writeHead(statusCode, forwardedResponseHeaders(upstreamResponse.headers)); } diff --git a/scripts/bench/tool-disclosure/report.ts b/scripts/bench/tool-disclosure/report.ts index 4e5cca1b2c..f607673858 100644 --- a/scripts/bench/tool-disclosure/report.ts +++ b/scripts/bench/tool-disclosure/report.ts @@ -9,6 +9,7 @@ import { type ToolDisclosureManifest, type ToolDisclosureSummary, } from "./types"; +import { validatePublicManifestFields } from "./validation"; export { buildConservativeClaims } from "./statistics"; @@ -202,6 +203,7 @@ function validateInputs( summary: ToolDisclosureSummary, artifacts: readonly EvidenceArtifact[], ): void { + validatePublicManifestFields(manifest); if ( manifest.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION || summary.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION diff --git a/scripts/bench/tool-disclosure/validation.ts b/scripts/bench/tool-disclosure/validation.ts index 86d61f8301..3d58d9bcf4 100644 --- a/scripts/bench/tool-disclosure/validation.ts +++ b/scripts/bench/tool-disclosure/validation.ts @@ -25,7 +25,76 @@ function sameJson(left: unknown, right: unknown): boolean { return JSON.stringify(left) === JSON.stringify(right); } +const SECRET_LIKE_COMPONENT = + /(?:^|[^a-z0-9])(?:api[-_ ]?keys?|tokens?|secrets?|passwords?|authorizations?)(?:$|[^a-z0-9])/iu; +const SECRET_LIKE_VALUE = /\b(?:bearer\s+\S+|(?:github_pat|ghp|gho|nvapi|sk)-[a-z0-9_-]{8,})\b/iu; +const URL_LIKE_VALUE = /\b[a-z][a-z0-9+.-]*:\/\/\S*/iu; +const ABSOLUTE_HOST_PATH = + /(?:^|[=\s"'(:,])(?:\/[a-z0-9._~-][^\s,;]*|[a-z]:[\\/][^\s,;]*|\\\\[^\\\s]+\\[^\s,;]*)/iu; +const PRIVATE_ROUTING_FLAGS = new Set([ + "--admin-key", + "--api-key", + "--base-url", + "--endpoint", + "--host", + "--mount", + "--port", + "--root-path", + "--socket", + "--ssl-ca-certs", + "--ssl-certfile", + "--ssl-keyfile", + "--uds", + "--unix-socket", + "--url", +]); + +function normalizedSecurityText(value: string): string { + return value.replace(/([a-z0-9])([A-Z])/gu, "$1_$2"); +} + +function validatePublicManifestValue(value: unknown): void { + if (typeof value === "string") { + const normalized = normalizedSecurityText(value); + if (SECRET_LIKE_COMPONENT.test(normalized) || SECRET_LIKE_VALUE.test(value)) { + throw new Error("public manifest contains a secret-like field name or value"); + } + if (URL_LIKE_VALUE.test(value)) { + throw new Error("public manifest contains a URL-looking value"); + } + if (value.trim() === "/" || ABSOLUTE_HOST_PATH.test(value)) { + throw new Error("public manifest contains an absolute host path"); + } + return; + } + if (Array.isArray(value)) { + for (const item of value) validatePublicManifestValue(item); + return; + } + if (!value || typeof value !== "object") return; + for (const [name, item] of Object.entries(value)) { + if (SECRET_LIKE_COMPONENT.test(normalizedSecurityText(name))) { + throw new Error("public manifest contains a secret-like field name or value"); + } + validatePublicManifestValue(item); + } +} + +/** Fail closed on obvious private material before a manifest can enter public artifacts. */ +export function validatePublicManifestFields(manifest: ToolDisclosureManifest): void { + for (const flag of manifest.inference.public_vllm_flags) { + for (const token of flag.trim().split(/\s+/u)) { + const name = token.match(/^--[a-z0-9][a-z0-9-]*/iu)?.[0].toLowerCase(); + if (name && PRIVATE_ROUTING_FLAGS.has(name)) { + throw new Error("public_vllm_flags contains a private routing flag"); + } + } + } + validatePublicManifestValue(manifest); +} + export function validateFrozenManifest(manifest: ToolDisclosureManifest): void { + validatePublicManifestFields(manifest); const requiredEnvironmentText = [ manifest.environment.operating_system, manifest.environment.architecture, diff --git a/test/bench/tool-disclosure-execute.test.ts b/test/bench/tool-disclosure-execute.test.ts index 38773ea0c6..6040a361ad 100644 --- a/test/bench/tool-disclosure-execute.test.ts +++ b/test/bench/tool-disclosure-execute.test.ts @@ -8,12 +8,13 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { - assertSetupRetryAllowed, type AttemptJournalEntry, + assertSetupRetryAllowed, type CampaignAttestation, classifyInvocationFailure, materializeAttemptJournal, nextSetupAttempt, + readAttestedVllmConfiguration, recoverAttemptJournal, selectInspectedContainer, } from "../../scripts/bench/tool-disclosure/execute"; @@ -123,6 +124,64 @@ describe("tool-disclosure attempt journal", () => { expect(selectInspectedContainer([expected, other], expected.Id)).toBeUndefined(); }); + it("attests vLLM arguments only from the intended process configuration", () => { + const expected: ToolDisclosureManifest["inference"] = { + api: "chat-completions", + model_id: "example/model", + model_revision: "revision-1", + container_image: "registry.example/vllm:1.0.0", + container_digest: `sha256:${"a".repeat(64)}`, + vllm_version: "1.0.0", + tool_call_parser: "tool-parser", + reasoning_parser: "reasoning-parser", + temperature: 0, + concurrency: 1, + prefix_caching_enabled: false, + public_vllm_flags: ["--enable-auto-tool-choice", "--max-model-len 8192"], + }; + const command = + "vllm serve example/model --revision revision-1 " + + "--tool-call-parser tool-parser --reasoning-parser reasoning-parser " + + "--enable-auto-tool-choice --max-model-len 8192"; + expect( + readAttestedVllmConfiguration( + { + Config: { + Entrypoint: ["/bin/bash"], + Cmd: ["-lc", command], + Env: ["VLLM_USE_V1=1", "UNRELATED=ignored"], + }, + }, + expected, + ), + ).toEqual({ + cmd: ["-lc", command], + entrypoint: ["/bin/bash"], + env: ["VLLM_USE_V1=1"], + }); + + const decoy = [ + expected.model_id, + expected.model_revision, + expected.tool_call_parser, + expected.reasoning_parser, + ...expected.public_vllm_flags, + ].join(" "); + expect( + readAttestedVllmConfiguration( + { + Config: { + Entrypoint: ["vllm"], + Cmd: ["serve", "different/model"], + Env: [`UNRELATED=${decoy}`], + Labels: { unrelated: decoy }, + }, + }, + expected, + ), + ).toBeUndefined(); + }); + it("recovers one trailing partial append and deterministically materializes evidence", () => { const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); try { diff --git a/test/bench/tool-disclosure-recorder.test.ts b/test/bench/tool-disclosure-recorder.test.ts index 0783632800..ba79a149e1 100644 --- a/test/bench/tool-disclosure-recorder.test.ts +++ b/test/bench/tool-disclosure-recorder.test.ts @@ -350,12 +350,12 @@ describe("tool-disclosure recording proxy", () => { expect(received).toHaveLength(1); }); - it("passes redirects through without following them or recording their bodies", async () => { + it("rejects upstream redirects without relaying Location or body secrets", async () => { let hits = 0; const upstream = await startUpstream((_request, response) => { hits += 1; response.writeHead(307, { - location: "/v1/redirect-target?token=location-secret", + location: "https://example.invalid/v1?token=location-secret", "content-type": "text/plain", }); response.end("redirect-response-secret"); @@ -365,18 +365,21 @@ describe("tool-disclosure recording proxy", () => { proxy.beginRun(); const response = await fetch(`${baseUrl}/v1/chat/completions`, { method: "POST", - redirect: "manual", headers: { "content-type": "application/json" }, body: '{"messages":[{"content":"request-secret"}]}', }); - expect(response.status).toBe(307); - expect(response.headers.get("location")).toContain("location-secret"); - expect(await response.text()).toBe("redirect-response-secret"); + expect(response.status).toBe(502); + expect(response.headers.get("location")).toBeNull(); + expect(await response.json()).toEqual({ error: "upstream redirect rejected" }); expect(hits).toBe(1); const [event] = proxy.endRun(); - expect(event.status_code).toBe(307); - expect(event.outcome).toBe("completed"); + expect(event).toMatchObject({ + status_code: 502, + first_byte_monotonic_ms: null, + outcome: "request-rejected", + error_reason: "proxy-failure", + }); expect(JSON.stringify(event)).not.toMatch( /location-secret|redirect-response-secret|request-secret/, ); diff --git a/test/bench/tool-disclosure-report.test.ts b/test/bench/tool-disclosure-report.test.ts index bc9aa0e837..0d6c15363f 100644 --- a/test/bench/tool-disclosure-report.test.ts +++ b/test/bench/tool-disclosure-report.test.ts @@ -452,6 +452,16 @@ describe("tool-disclosure Markdown report", () => { expect(markdown).not.toContain("GPU"); }); + it("rejects unsafe public manifest values before rendering the report", () => { + const manifest = makeManifest(); + const summary = buildToolDisclosureSummary(manifest, completeEvidence(), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + manifest.inference.public_vllm_flags = ["--host=http://127.0.0.1:8000"]; + + expect(() => renderToolDisclosureMarkdown(manifest, summary)).toThrow(/private routing flag/u); + }); + it("rejects artifact paths so host locations cannot enter the report", () => { const manifest = makeManifest(); const summary = buildToolDisclosureSummary(manifest, completeEvidence(), { diff --git a/test/bench/tool-disclosure-validation.test.ts b/test/bench/tool-disclosure-validation.test.ts index dc42408759..11a0377f2b 100644 --- a/test/bench/tool-disclosure-validation.test.ts +++ b/test/bench/tool-disclosure-validation.test.ts @@ -312,6 +312,51 @@ describe("tool-disclosure complete-evidence validation", () => { expect(() => validateFrozenManifest(arbitraryAccelerator)).toThrow(/inconsistent/u); }); + it("rejects secret-like names and values in the public manifest", () => { + for (const marker of ["api_key", "token", "secret", "password", "authorization"]) { + const secretName = structuredClone(fixture.manifest) as ToolDisclosureManifest & + Record; + secretName[marker] = "opaque-value"; + expect(() => validateFrozenManifest(secretName)).toThrow(/secret-like/u); + + const secretValue = structuredClone(fixture.manifest); + secretValue.environment.cpu_model = `${marker}=opaque-value`; + expect(() => validateFrozenManifest(secretValue)).toThrow(/secret-like/u); + } + }); + + it("rejects URL-looking values and absolute host paths in the public manifest", () => { + const endpoint = structuredClone(fixture.manifest); + endpoint.inference.model_revision = "https://example.invalid/private-revision"; + expect(() => validateFrozenManifest(endpoint)).toThrow(/URL-looking/u); + + const posixPath = structuredClone(fixture.manifest); + posixPath.environment.cpu_model = "/home/user/private-model"; + expect(() => validateFrozenManifest(posixPath)).toThrow(/absolute host path/u); + + const windowsPath = structuredClone(fixture.manifest); + windowsPath.environment.cpu_model = "C:\\Users\\user\\private-model"; + expect(() => validateFrozenManifest(windowsPath)).toThrow(/absolute host path/u); + }); + + it("rejects private routing flags and accepts reviewed public vLLM flags", () => { + for (const flag of [ + "--api-key=secret", + "--host=http://127.0.0.1:8000", + "--mount=/home/user/model", + "--port=8000", + "--enable-prefix-caching --host=0.0.0.0", + ]) { + const unsafe = structuredClone(fixture.manifest); + unsafe.inference.public_vllm_flags = [flag]; + expect(() => validateFrozenManifest(unsafe)).toThrow(/private routing flag/u); + } + + const reviewed = structuredClone(fixture.manifest); + reviewed.inference.public_vllm_flags = ["--enable-prefix-caching", "--max-model-len=8192"]; + expect(() => validateFrozenManifest(reviewed)).not.toThrow(); + }); + it("accepts the exact 1,884-run frozen matrix and rejects reordered or tampered evidence", { timeout: 30_000, }, () => { From 20a64a8c52a705107744a44ff92f5aaf111e40b5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 16:39:08 -0700 Subject: [PATCH 09/31] fix(bench): pin localhost to literal loopback Signed-off-by: Aaron Erickson --- scripts/bench/tool-disclosure/recorder.ts | 4 +++ scripts/bench/tool-disclosure/telemetry.ts | 1 + test/bench/tool-disclosure-recorder.test.ts | 23 +++++++++++++++ test/bench/tool-disclosure-telemetry.test.ts | 31 ++++++++++++++++---- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/scripts/bench/tool-disclosure/recorder.ts b/scripts/bench/tool-disclosure/recorder.ts index 93bafbe1bd..93f856f1f9 100644 --- a/scripts/bench/tool-disclosure/recorder.ts +++ b/scripts/bench/tool-disclosure/recorder.ts @@ -199,6 +199,10 @@ function parseUpstreamBaseUrl(value: string): URL { if (url.search || url.hash) { throw new Error("upstreamBaseUrl must not contain a query or fragment"); } + // Do not delegate the accepted localhost alias to ambient DNS resolution. + if (hostname === "localhost") { + url.hostname = LOOPBACK_HOST; + } return url; } diff --git a/scripts/bench/tool-disclosure/telemetry.ts b/scripts/bench/tool-disclosure/telemetry.ts index f1282edb13..7085b84c11 100644 --- a/scripts/bench/tool-disclosure/telemetry.ts +++ b/scripts/bench/tool-disclosure/telemetry.ts @@ -42,6 +42,7 @@ export function validatedTelemetryUrl(raw: string, endpoint: string): URL { if (!isLoopback(url.hostname)) { throw new Error("vLLM telemetry is allowed only on loopback"); } + if (url.hostname.toLowerCase() === "localhost") url.hostname = "127.0.0.1"; url.username = ""; url.password = ""; url.search = ""; diff --git a/test/bench/tool-disclosure-recorder.test.ts b/test/bench/tool-disclosure-recorder.test.ts index ba79a149e1..93a7977376 100644 --- a/test/bench/tool-disclosure-recorder.test.ts +++ b/test/bench/tool-disclosure-recorder.test.ts @@ -132,6 +132,29 @@ describe("tool-disclosure recording proxy", () => { ).toThrow("upstreamBaseUrl is allowed only on loopback"); }); + it("normalizes localhost to literal IPv4 before connecting upstream", async () => { + let observedHost = ""; + let observedPath = ""; + const upstream = await startUpstream((request, response) => { + observedHost = request.headers.host ?? ""; + observedPath = request.url ?? ""; + response.end("{}"); + }); + const upstreamUrl = new URL(upstream.baseUrl); + upstreamUrl.hostname = "localhost"; + upstreamUrl.pathname = "/proxy/v1"; + const { proxy, baseUrl } = await startProxy(upstreamUrl.toString()); + + proxy.beginRun("localhost-normalization"); + const response = await fetch(`${baseUrl}/v1/models`); + await response.arrayBuffer(); + proxy.endRun(); + + expect(response.status).toBe(200); + expect(observedHost).toBe(`127.0.0.1:${upstreamUrl.port}`); + expect(observedPath).toBe("/proxy/v1/models"); + }); + it("accepts bounded public scheduled run IDs and rejects unsafe IDs", () => { const proxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl: "https://localhost:8000/v1", diff --git a/test/bench/tool-disclosure-telemetry.test.ts b/test/bench/tool-disclosure-telemetry.test.ts index 6b0a1b83e0..0c671a049a 100644 --- a/test/bench/tool-disclosure-telemetry.test.ts +++ b/test/bench/tool-disclosure-telemetry.test.ts @@ -7,6 +7,7 @@ import { countTokensWithVllm, parseProcessStartTime, parseVllmTokenMetrics, + readVllmProcessStartTime, tokenDelta, validatedTelemetryUrl, } from "../../scripts/bench/tool-disclosure/telemetry"; @@ -53,24 +54,42 @@ vllm:generation_tokens_total{model_name="bench",engine="1"} 2 "vLLM telemetry is allowed only on loopback", ); expect(validatedTelemetryUrl("https://localhost:8000/v1", "/metrics").toString()).toBe( - "https://localhost:8000/metrics", + "https://127.0.0.1:8000/metrics", ); expect(validatedTelemetryUrl("https://127.0.0.1:8000/v1", "/tokenize").toString()).toBe( "https://127.0.0.1:8000/tokenize", ); + expect(validatedTelemetryUrl("http://127.0.0.2:8000/v1", "/metrics").toString()).toBe( + "http://127.0.0.2:8000/metrics", + ); expect(validatedTelemetryUrl("https://[::1]:8000/v1", "/metrics").toString()).toBe( "https://[::1]:8000/metrics", ); }); - it("reads token count variants without exposing response bodies", async () => { - const fetchImpl = (async () => - new Response(JSON.stringify({ tokens: [1, 2, 3, 4] }), { + it("fetches localhost metrics and tokenizer routes through literal IPv4 loopback", async () => { + const requestedUrls: string[] = []; + const metricsFetch = (async (input: string | URL | Request) => { + requestedUrls.push(input.toString()); + return new Response("process_start_time_seconds 1712345678\n", { status: 200 }); + }) as typeof fetch; + const tokenizerFetch = (async (input: string | URL | Request) => { + requestedUrls.push(input.toString()); + return new Response(JSON.stringify({ tokens: [1, 2, 3, 4] }), { status: 200, headers: { "content-type": "application/json" }, - })) as typeof fetch; + }); + }) as typeof fetch; + + await expect(readVllmProcessStartTime("https://localhost:8443/v1", metricsFetch)).resolves.toBe( + 1712345678, + ); await expect( - countTokensWithVllm("http://localhost:8000", "bench-model", "[]", fetchImpl), + countTokensWithVllm("http://localhost:8000", "bench-model", "[]", tokenizerFetch), ).resolves.toBe(4); + expect(requestedUrls).toEqual([ + "https://127.0.0.1:8443/metrics", + "http://127.0.0.1:8000/tokenize", + ]); }); }); From abcfec014d48b72a37f381e9d2a95e905390f9bd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 19:39:45 -0700 Subject: [PATCH 10/31] test: add hosted tool disclosure smoke Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 100 ++++++- .../progressive-tool-disclosure-benchmark.mdx | 4 + scripts/bench/tool-disclosure/drivers.ts | 43 ++- test/bench/tool-disclosure-drivers.test.ts | 28 ++ test/e2e/live/tool-disclosure-smoke.test.ts | 250 ++++++++++++++++++ .../support/jetson-workflow-boundary.test.ts | 2 +- ...disclosure-smoke-workflow-boundary.test.ts | 130 +++++++++ ...ad-e2e-artifacts-workflow-boundary.test.ts | 6 +- ...upload-e2e-artifacts-workflow-boundary.mts | 4 +- 9 files changed, 552 insertions(+), 15 deletions(-) create mode 100644 test/e2e/live/tool-disclosure-smoke.test.ts create mode 100644 test/e2e/support/tool-disclosure-smoke-workflow-boundary.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8d65390489..f118bffdfb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -14,7 +14,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." + description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, tool-disclosure-smoke, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." required: false default: "" type: string @@ -723,6 +723,96 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + tool-disclosure-smoke: + needs: generate-matrix + # This smoke provisions real agent sandboxes and consumes hosted inference. + # Keep it opt-in: it validates benchmark plumbing on ordinary CI hardware, + # but it is not the complete two-campaign claim-validation protocol. + if: ${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-smoke,') }} + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 60 + env: + E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" + E2E_TARGET_ID: "tool-disclosure-smoke" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-smoke + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_OPENSHELL_CHANNEL: stable + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-ultra + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-ultra + NEMOCLAW_PREFERRED_API: openai-completions + OPENSHELL_GATEWAY: nemoclaw + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - name: Install and verify cloudflared prerequisite + env: + CLOUDFLARED_VERSION: "2026.6.1" + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" + run: | + set -euo pipefail + cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" + curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" + printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - + package="$(dpkg-deb -f "${cloudflared_deb}" Package)" + version="$(dpkg-deb -f "${cloudflared_deb}" Version)" + architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" + if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then + printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 + exit 1 + fi + sudo dpkg -i "${cloudflared_deb}" + cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" + + - name: Install OpenShell CLI + env: + NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" + run: bash scripts/install-openshell.sh + + - name: Run tool-disclosure smoke live test + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-live \ + test/e2e/live/tool-disclosure-smoke.test.ts \ + --silent=false --reporter=default + + - name: Upload tool-disclosure smoke artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + onboard-negative-paths: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.targets), ',onboard-negative-paths,') }} @@ -4570,6 +4660,7 @@ jobs: openshell-gateway-auth-contract, mcp-bridge, mcp-bridge-dev, + tool-disclosure-smoke, onboard-negative-paths, skill-agent, openclaw-skill-cli, @@ -4674,6 +4765,11 @@ jobs: target: 'mcp-bridge-dev', reason: 'default dispatch excludes moving OpenShell dev artifacts unless explicitly selected', }, + 'tool-disclosure-smoke': { + job: 'tool-disclosure-smoke', + target: 'tool-disclosure-smoke', + reason: 'default dispatch excludes the hosted-inference benchmark smoke unless explicitly selected', + }, 'jetson-nvmap-gpu': { job: 'jetson-nvmap-gpu', target: 'jetson-nvmap-gpu', @@ -4804,7 +4900,7 @@ jobs: ? '**Requested jobs:** _(selector rejected by workflow validation)_' : requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` - : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', + : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `tool-disclosure-smoke`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`, '', '| Job | Result |', diff --git a/docs/inference/progressive-tool-disclosure-benchmark.mdx b/docs/inference/progressive-tool-disclosure-benchmark.mdx index d94a5cd2bb..c2f19086b1 100644 --- a/docs/inference/progressive-tool-disclosure-benchmark.mdx +++ b/docs/inference/progressive-tool-disclosure-benchmark.mdx @@ -17,6 +17,10 @@ The repository does not include measured results. Run both campaigns and retain their evidence before you make a performance or quality claim. +The explicit-only `tool-disclosure-smoke` E2E job is a smaller operational check for ordinary GitHub-hosted runners. +It runs one frozen task in progressive and direct modes with fresh Deep Agents sandboxes and hosted inference. +Its output validates live wiring only: it is not a complete campaign, cannot be summarized by this protocol, and does not support benchmark claims. + ## Understand the frozen protocol The preparation command generates one deterministic catalog, task corpus, and paired execution schedule. diff --git a/scripts/bench/tool-disclosure/drivers.ts b/scripts/bench/tool-disclosure/drivers.ts index 409b408085..a05ed996d5 100644 --- a/scripts/bench/tool-disclosure/drivers.ts +++ b/scripts/bench/tool-disclosure/drivers.ts @@ -115,6 +115,41 @@ function record(value: unknown): Record | null { : null; } +function stripTerminalControls(value: string): string { + return value.replace(/\u001b\[[0-?]*[ -/]*[@-~]/gu, ""); +} + +/** + * Deep Agents Code prints the final assistant block immediately before its + * `Task completed` marker, followed by an `Agent active` timing line. Tool + * trace lines precede that block. Select only the delimited assistant block so + * neither a tool trace nor a trailing status line can satisfy an oracle. + */ +function extractDeepAgentsFinalAssistantOutput(raw: string): string { + const lines = stripTerminalControls(raw).split(/\r?\n/u); + let completed = -1; + for (let index = lines.length - 1; index >= 0; index -= 1) { + if (/^(?:[✓✔]\s*)?Task completed\b/iu.test(lines[index].trim())) { + completed = index; + break; + } + } + if (completed < 0) return ""; + + let boundary = -1; + for (let index = 0; index < completed; index += 1) { + const line = lines[index].trim(); + if (/^(?:🔧\s*)?Calling tool:/u.test(line) || /^(?:[✓✔]\s*)?Server ready\s*$/iu.test(line)) { + boundary = index; + } + } + if (boundary < 0) return ""; + return lines + .slice(boundary + 1, completed) + .join("\n") + .trim(); +} + /** Extract only the user-visible final assistant field, never raw tool traces. */ export function extractFinalAssistantOutput(agent: ToolDisclosureAgent, raw: string): string { if (agent === "hermes") { @@ -140,13 +175,7 @@ export function extractFinalAssistantOutput(agent: ToolDisclosureAgent, raw: str } return parts.join("\n"); } - return ( - raw - .split(/\r?\n/u) - .map((line) => line.trim()) - .filter(Boolean) - .at(-1) ?? "" - ); + return extractDeepAgentsFinalAssistantOutput(raw); } export function buildOpenClawCallLogCommand(options: { diff --git a/test/bench/tool-disclosure-drivers.test.ts b/test/bench/tool-disclosure-drivers.test.ts index 9284f346df..8618a4925c 100644 --- a/test/bench/tool-disclosure-drivers.test.ts +++ b/test/bench/tool-disclosure-drivers.test.ts @@ -98,4 +98,32 @@ describe("tool-disclosure agent drivers", () => { ), ).toBe("FINAL_NONCE"); }); + + it("extracts the Deep Agents final block before trailing task status", () => { + const output = [ + "Running task non-interactively...", + "App: v0.1.30 | Agent: agent (default) | Model: example/model | Thread: thread-1", + "Starting LangGraph server...", + "✓ Loaded 1 MCP tool", + "✓ Server ready", + "🔧 Calling tool: search_tools (trace: TOOL_ONLY_NONCE)", + "🔧 Calling tool: benchmark_echo", + "Operation complete.", + "FINAL_NONCE", + "", + "✓ Task completed", + "", + "Agent active 1.3s", + ].join("\n"); + + expect(extractFinalAssistantOutput("langchain-deepagents-code", output)).toBe( + "Operation complete.\nFINAL_NONCE", + ); + expect( + extractFinalAssistantOutput( + "langchain-deepagents-code", + "✓ Server ready\n🔧 Calling tool: benchmark_echo (result: TOOL_ONLY_NONCE)\n✓ Task completed\nAgent active 1.3s", + ), + ).toBe(""); + }); }); diff --git a/test/e2e/live/tool-disclosure-smoke.test.ts b/test/e2e/live/tool-disclosure-smoke.test.ts new file mode 100644 index 0000000000..255afc069c --- /dev/null +++ b/test/e2e/live/tool-disclosure-smoke.test.ts @@ -0,0 +1,250 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; + +import { + DEFAULT_SYNTHETIC_CATALOG_SEED, + generateCatalogPrefix, + generateSyntheticCatalog, +} from "../../../scripts/bench/tool-disclosure/catalog"; +import { + buildAgentDriverCommand, + extractFinalAssistantOutput, +} from "../../../scripts/bench/tool-disclosure/drivers"; +import { gradeTaskRun } from "../../../scripts/bench/tool-disclosure/grading"; +import { SyntheticMcpServer } from "../../../scripts/bench/tool-disclosure/mcp-server"; +import { startQuickTunnel } from "../../../scripts/bench/tool-disclosure/quick-tunnel"; +import type { ToolDisclosureMode } from "../../../scripts/bench/tool-disclosure/schedule"; +import { generatePrimaryTaskSet } from "../../../scripts/bench/tool-disclosure/tasks"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; +import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; + +const AGENT = "langchain-deepagents-code" as const; +const CATALOG_SIZE = 64; +const TASK_ID = "primary-single-01"; +const MODES = ["progressive", "direct"] as const satisfies readonly ToolDisclosureMode[]; +const MCP_SERVER_NAME = "benchmark"; +const MCP_TOKEN_ENV = "NEMOCLAW_BENCHMARK_MCP_BEARER"; +const TEST_TIMEOUT_MS = 50 * 60_000; + +function sandboxName(mode: ToolDisclosureMode): string { + return `e2e-tool-disclosure-${mode}`; +} + +function hostEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +function gitSha(): string { + const sha = + process.env.GITHUB_SHA?.trim() || + execFileSync("git", ["rev-parse", "HEAD"], { + encoding: "utf8", + }).trim(); + if (!/^[a-f0-9]{40}$/u.test(sha)) throw new Error("tool-disclosure smoke requires a git SHA"); + return sha; +} + +test.skipIf(!shouldRunLiveE2E())( + "tool disclosure hosted-inference smoke completes one frozen task in both modes", + { timeout: TEST_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets }) => { + const hosted = requireHostedInferenceConfig(secrets); + const bearerToken = randomBytes(32).toString("hex"); + artifacts.addRedactionValues([hosted.apiKey, bearerToken]); + + const catalog = generateSyntheticCatalog({ seed: DEFAULT_SYNTHETIC_CATALOG_SEED }); + const catalogPrefix = generateCatalogPrefix(catalog, CATALOG_SIZE); + const primaryTasks = generatePrimaryTaskSet(catalog); + const task = primaryTasks.tasks.find((candidate) => candidate.id === TASK_ID); + if (!task) throw new Error(`frozen task ${TASK_ID} is missing`); + if (task.min_catalog_size > catalogPrefix.size) { + throw new Error(`frozen task ${TASK_ID} does not fit the smoke catalog`); + } + + await artifacts.writeJson("scenario.json", { + id: "tool-disclosure-smoke", + boundary: "two real Deep Agents Code sandboxes with hosted inference and synthetic MCP", + claim_eligible: false, + agent: AGENT, + modes: MODES, + catalog_size: CATALOG_SIZE, + task_id: TASK_ID, + model_id: hosted.model, + }); + + const mcp = new SyntheticMcpServer(catalogPrefix, bearerToken); + const mcpAddress = await mcp.start(); + cleanup.add("stop tool-disclosure synthetic MCP server", () => mcp.stop()); + const tunnel = await startQuickTunnel({ port: mcpAddress.port }); + cleanup.add("stop tool-disclosure MCP quick tunnel", () => tunnel.close()); + + for (const mode of MODES) { + const name = sandboxName(mode); + cleanup.add(`destroy ${mode} tool-disclosure smoke sandbox`, () => + host.bestEffortCleanupSandbox(name, { + artifactName: `cleanup-${mode}-tool-disclosure-sandbox`, + env: hostEnv(), + }), + ); + await host.cleanupSandbox(name, { + artifactName: `preclean-${mode}-tool-disclosure-sandbox`, + env: hostEnv(), + }); + + const onboard = await host.nemoclaw( + [ + "onboard", + "--non-interactive", + "--fresh", + "--yes", + "--yes-i-accept-third-party-software", + "--agent", + AGENT, + "--name", + name, + "--tool-disclosure", + mode, + ], + { + artifactName: `onboard-tool-disclosure-${mode}`, + env: hostEnv({ + ...hosted.env, + NEMOCLAW_AGENT: AGENT, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: name, + NEMOCLAW_TOOL_DISCLOSURE: mode, + }), + redactionValues: [hosted.apiKey], + timeoutMs: 20 * 60_000, + }, + ); + expect(onboard.exitCode, resultText(onboard)).toBe(0); + + const attestation = await sandbox.exec( + name, + ["sh", "-c", `test "$NEMOCLAW_TOOL_DISCLOSURE" = ${JSON.stringify(mode)}`], + { + artifactName: `attest-tool-disclosure-${mode}`, + env: sandboxAccessEnv(), + timeoutMs: 30_000, + }, + ); + expect(attestation.exitCode, resultText(attestation)).toBe(0); + } + + for (const mode of MODES) { + const add = await host.nemoclaw( + [ + sandboxName(mode), + "mcp", + "add", + MCP_SERVER_NAME, + "--url", + tunnel.mcpUrl, + "--env", + MCP_TOKEN_ENV, + ], + { + artifactName: `add-tool-disclosure-mcp-${mode}`, + env: hostEnv({ [MCP_TOKEN_ENV]: bearerToken }), + redactionValues: [bearerToken], + timeoutMs: 5 * 60_000, + }, + ); + expect(add.exitCode, resultText(add)).toBe(0); + } + + const results = []; + for (const mode of MODES) { + const runId = `smoke-${AGENT}-${mode}-${TASK_ID}`; + const driver = buildAgentDriverCommand({ + openshellBin: process.env.OPENSHELL_BIN, + sandboxName: sandboxName(mode), + agent: AGENT, + prompt: task.prompt, + sessionId: runId, + }); + mcp.beginRun(runId); + const startedAt = process.hrtime.bigint(); + let invocation; + let calls; + try { + invocation = await host.command(driver.command, driver.args, { + artifactName: `invoke-tool-disclosure-${mode}`, + env: hostEnv(), + redactionValues: [...driver.redactions, hosted.apiKey, bearerToken], + timeoutMs: 10 * 60_000, + }); + } finally { + calls = mcp.endRun(); + } + const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); + const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); + const graded = gradeTaskRun(task, calls, finalOutput); + const outcome = invocation.timedOut + ? "timeout" + : invocation.exitCode === 0 + ? graded.outcome + : "model-error"; + results.push({ + run_id: runId, + mode, + outcome, + invocation: { + exit_code: invocation.exitCode, + timed_out: invocation.timedOut, + elapsed_ms: elapsedMs, + }, + synthetic_call_count: calls.length, + correctness: graded.correctness, + }); + } + + await artifacts.writeJson("tool-disclosure-smoke.json", { + schema_version: "nemoclaw.tool_disclosure_smoke.v1", + generated_at: new Date().toISOString(), + claim_eligible: false, + sut_git_sha: gitSha(), + profile: { + agent: AGENT, + modes: MODES, + catalog_seed: catalog.seed, + catalog_size: CATALOG_SIZE, + catalog_tools_sha256: catalogPrefix.tools_sha256, + task_id: task.id, + task_kind: task.kind, + task_set_sha256: primaryTasks.tasks_sha256, + repetitions_per_mode: 1, + observed_runs: results.length, + }, + inference: { + api: "openai-compatible", + model_id: hosted.model, + }, + results, + limitations: [ + "This smoke verifies live wiring and task completion; it is not the full benchmark.", + "One observation per mode is insufficient for performance or quality claims.", + "The fixed progressive-then-direct order makes elapsed times informational and subject to cold-cache and ordering effects.", + "The smoke does not collect vLLM tokenizer, token-counter, or request-recorder evidence.", + ], + }); + + expect(results).toHaveLength(2); + for (const result of results) { + expect(result.outcome, `${result.mode} smoke outcome`).toBe("success"); + expect(result.correctness.task_success, `${result.mode} task correctness`).toBe(true); + expect(result.synthetic_call_count, `${result.mode} synthetic call count`).toBe(1); + } + }, +); diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 0bd9353938..8dd50171a2 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -23,7 +23,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { expect(inventory.allowedJobs).toContain("jetson-nvmap-gpu"); expect(inventory.explicitOnlyJobs).toContain("jetson-nvmap-gpu"); expect(formatFreeStandingJobsInventoryForShell(inventory)).toContain( - "explicit_only_jobs_csv=openshell-gateway-auth-contract,mcp-bridge-dev,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", + "explicit_only_jobs_csv=openshell-gateway-auth-contract,mcp-bridge-dev,tool-disclosure-smoke,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", ); expect(inventory.targetToJob.get("jetson-nvmap-gpu")).toBe("jetson-nvmap-gpu"); expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( diff --git a/test/e2e/support/tool-disclosure-smoke-workflow-boundary.test.ts b/test/e2e/support/tool-disclosure-smoke-workflow-boundary.test.ts new file mode 100644 index 0000000000..23405e9cd0 --- /dev/null +++ b/test/e2e/support/tool-disclosure-smoke-workflow-boundary.test.ts @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { UPLOAD_E2E_ARTIFACTS_ACTION } from "../../../tools/e2e/upload-e2e-artifacts-workflow-boundary.mts"; +import { + evaluateE2eWorkflowDispatchSelectors, + readFreeStandingJobsInventory, + validateE2eWorkflowBoundary, +} from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +type WorkflowStep = { + env?: Record; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type WorkflowJob = { + env?: Record; + if?: string; + needs?: string | string[]; + permissions?: Record; + "runs-on"?: string; + steps?: WorkflowStep[]; + "timeout-minutes"?: number; +}; + +function smokeJob(): WorkflowJob { + const workflow = readWorkflow() as { jobs: Record }; + const job = workflow.jobs["tool-disclosure-smoke"]; + expect(job).toBeDefined(); + return job; +} + +function namedStep(job: WorkflowJob, name: string): WorkflowStep | undefined { + return job.steps?.find((step) => step.name === name); +} + +describe("tool-disclosure smoke workflow boundary", () => { + it("keeps the hosted-inference smoke explicit-only", () => { + const inventory = readFreeStandingJobsInventory(); + + expect(validateE2eWorkflowBoundary()).toEqual([]); + expect(inventory.allowedJobs).toContain("tool-disclosure-smoke"); + expect(inventory.explicitOnlyJobs).toContain("tool-disclosure-smoke"); + expect(inventory.targetToJob.get("tool-disclosure-smoke")).toBe("tool-disclosure-smoke"); + expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( + "tool-disclosure-smoke", + ); + + for (const selector of [ + { jobs: "tool-disclosure-smoke" }, + { targets: "tool-disclosure-smoke" }, + ]) { + expect(evaluateE2eWorkflowDispatchSelectors(selector)).toMatchObject({ + valid: true, + liveTargetsRun: false, + selectedFreeStandingJobs: ["tool-disclosure-smoke"], + registryTargets: [], + }); + } + }); + + it("runs the focused live test with reviewed prerequisites and scoped secrets", () => { + const job = smokeJob(); + expect(job).toMatchObject({ + needs: "generate-matrix", + if: "${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-smoke,') }}", + "runs-on": "ubuntu-latest", + permissions: { contents: "read" }, + "timeout-minutes": 60, + env: { + E2E_JOB: "1", + E2E_DEFAULT_ENABLED: "0", + E2E_TARGET_ID: "tool-disclosure-smoke", + E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-smoke", + NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "stable", + OPENSHELL_GATEWAY: "nemoclaw", + }, + }); + expect(job.env).not.toHaveProperty("NVIDIA_INFERENCE_API_KEY"); + + const checkout = job.steps?.find((step) => step.uses?.startsWith("actions/checkout@")); + expect(checkout?.with?.["persist-credentials"]).toBe(false); + expect(namedStep(job, "Authenticate to Docker Hub")).toBeDefined(); + expect(namedStep(job, "Prepare E2E workspace")?.uses).toBe( + "NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28", + ); + + const cloudflared = namedStep(job, "Install and verify cloudflared prerequisite"); + expect(cloudflared?.env).toEqual({ + CLOUDFLARED_VERSION: "2026.6.1", + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526", + }); + expect(cloudflared?.run).toContain("sha256sum -c -"); + expect(cloudflared?.run).toContain("dpkg-deb -f"); + + const run = namedStep(job, "Run tool-disclosure smoke live test"); + expect(run?.env).toEqual({ + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + }); + expect(run?.run).toContain("test/e2e/live/tool-disclosure-smoke.test.ts"); + expect(run?.run).toContain("--project e2e-live"); + }); + + it("maps only the smoke artifact directory through the reviewed uploader", () => { + const job = smokeJob(); + const upload = namedStep(job, "Upload tool-disclosure smoke artifacts"); + expect(upload).toEqual({ + name: "Upload tool-disclosure smoke artifacts", + if: "always()", + uses: UPLOAD_E2E_ARTIFACTS_ACTION, + }); + expect(job.env?.E2E_TARGET_ID).toBe("tool-disclosure-smoke"); + expect(job.env?.E2E_ARTIFACT_DIR).toBe( + "${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-smoke", + ); + expect(job.steps?.at(-1)?.name).toBe("Clean up Docker auth"); + }); +}); diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 3c67b3d878..d345dd66a1 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -77,7 +77,7 @@ function validateActionMutation(mutate: (action: MutableAction) => void): string } describe("upload-e2e-artifacts workflow boundary", () => { - it("binds one canonical uploader to all 73 E2E execution jobs", () => { + it("binds one canonical uploader to all 74 E2E execution jobs", () => { expect(validateUploadE2eArtifactsAction()).toEqual([]); expect(validateUploadE2eArtifactsInvocations(readWorkflow())).toEqual([]); }); @@ -177,8 +177,8 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "upload-e2e-artifacts must cover exactly 73 live and E2E_JOB execution jobs", - "upload-e2e-artifacts must keep exactly 62 default callers", + "upload-e2e-artifacts must cover exactly 74 live and E2E_JOB execution jobs", + "upload-e2e-artifacts must keep exactly 63 default callers", ]), ); }); diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index ef42515dda..0b3914c63a 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -34,8 +34,8 @@ const CALLER_ALWAYS = "always()"; const MCP_SCANNED_UPLOAD_CONDITION = "${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}"; const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; -const EXPECTED_UPLOAD_JOB_COUNT = 73; -const EXPECTED_DEFAULT_CALLER_COUNT = 62; +const EXPECTED_UPLOAD_JOB_COUNT = 74; +const EXPECTED_DEFAULT_CALLER_COUNT = 63; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { From 6b23a4d84c4a05100a5c7eb84ad810446be7eb4a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 19:50:38 -0700 Subject: [PATCH 11/31] refactor: call tool disclosure a performance test Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 32 +++++----- docs/index.yml | 12 ++-- ...sive-tool-disclosure-performance-test.mdx} | 46 +++++++------- package.json | 2 +- scripts/bench/README.md | 59 ------------------ scripts/performance/README.md | 62 +++++++++++++++++++ .../tool-disclosure/artifacts.ts | 13 ++-- .../tool-disclosure/assemble-run.ts | 18 +++--- .../tool-disclosure/catalog.ts | 25 ++++---- .../tool-disclosure/drivers.ts | 9 +-- .../tool-disclosure/execute.ts | 20 +++--- .../tool-disclosure/grading.ts | 8 +-- .../tool-disclosure/mcp-server.ts | 22 +++---- .../tool-disclosure/openclaw-fixture.ts | 18 +++--- .../tool-disclosure/quick-tunnel.ts | 0 .../tool-disclosure/recorder.ts | 8 +-- .../tool-disclosure/report.ts | 18 +++--- .../tool-disclosure/run.ts | 36 ++++++----- .../tool-disclosure/schedule.ts | 0 .../tool-disclosure/statistics.ts | 20 +++--- .../tool-disclosure/subprocess.ts | 4 +- .../tool-disclosure/tasks.ts | 49 ++++++++------- .../tool-disclosure/telemetry.ts | 0 .../tool-disclosure/types.ts | 54 +++++++++------- .../tool-disclosure/validation.ts | 8 +-- ...tool-disclosure-performance-smoke.test.ts} | 57 +++++++++-------- .../support/jetson-workflow-boundary.test.ts | 2 +- ...rformance-smoke-workflow-boundary.test.ts} | 45 +++++++------- .../tool-disclosure-catalog.test.ts | 8 +-- .../tool-disclosure-drivers.test.ts | 34 +++++----- .../tool-disclosure-execute.test.ts | 8 +-- .../tool-disclosure-fixture.test.ts | 32 +++++----- .../tool-disclosure-mcp-server.test.ts | 8 +-- .../tool-disclosure-quick-tunnel.test.ts | 8 ++- .../tool-disclosure-recorder.test.ts | 4 +- .../tool-disclosure-report.test.ts | 24 +++---- .../tool-disclosure-schedule.test.ts | 6 +- .../tool-disclosure-telemetry.test.ts | 16 ++--- .../tool-disclosure-validation.test.ts | 28 ++++----- 39 files changed, 426 insertions(+), 397 deletions(-) rename docs/inference/{progressive-tool-disclosure-benchmark.mdx => progressive-tool-disclosure-performance-test.mdx} (86%) create mode 100644 scripts/performance/README.md rename scripts/{bench => performance}/tool-disclosure/artifacts.ts (86%) rename scripts/{bench => performance}/tool-disclosure/assemble-run.ts (91%) rename scripts/{bench => performance}/tool-disclosure/catalog.ts (95%) rename scripts/{bench => performance}/tool-disclosure/drivers.ts (94%) rename scripts/{bench => performance}/tool-disclosure/execute.ts (97%) rename scripts/{bench => performance}/tool-disclosure/grading.ts (90%) rename scripts/{bench => performance}/tool-disclosure/mcp-server.ts (89%) rename scripts/{bench => performance}/tool-disclosure/openclaw-fixture.ts (90%) rename scripts/{bench => performance}/tool-disclosure/quick-tunnel.ts (100%) rename scripts/{bench => performance}/tool-disclosure/recorder.ts (98%) rename scripts/{bench => performance}/tool-disclosure/report.ts (94%) rename scripts/{bench => performance}/tool-disclosure/run.ts (94%) rename scripts/{bench => performance}/tool-disclosure/schedule.ts (100%) rename scripts/{bench => performance}/tool-disclosure/statistics.ts (97%) rename scripts/{bench => performance}/tool-disclosure/subprocess.ts (95%) rename scripts/{bench => performance}/tool-disclosure/tasks.ts (90%) rename scripts/{bench => performance}/tool-disclosure/telemetry.ts (100%) rename scripts/{bench => performance}/tool-disclosure/types.ts (84%) rename scripts/{bench => performance}/tool-disclosure/validation.ts (98%) rename test/e2e/live/{tool-disclosure-smoke.test.ts => tool-disclosure-performance-smoke.test.ts} (76%) rename test/e2e/support/{tool-disclosure-smoke-workflow-boundary.test.ts => tool-disclosure-performance-smoke-workflow-boundary.test.ts} (72%) rename test/{bench => performance}/tool-disclosure-catalog.test.ts (97%) rename test/{bench => performance}/tool-disclosure-drivers.test.ts (76%) rename test/{bench => performance}/tool-disclosure-execute.test.ts (96%) rename test/{bench => performance}/tool-disclosure-fixture.test.ts (90%) rename test/{bench => performance}/tool-disclosure-mcp-server.test.ts (93%) rename test/{bench => performance}/tool-disclosure-quick-tunnel.test.ts (86%) rename test/{bench => performance}/tool-disclosure-recorder.test.ts (99%) rename test/{bench => performance}/tool-disclosure-report.test.ts (95%) rename test/{bench => performance}/tool-disclosure-schedule.test.ts (95%) rename test/{bench => performance}/tool-disclosure-telemetry.test.ts (85%) rename test/{bench => performance}/tool-disclosure-validation.test.ts (93%) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f118bffdfb..57421ed77d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -14,7 +14,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, tool-disclosure-smoke, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." + description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, tool-disclosure-performance-smoke, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." required: false default: "" type: string @@ -723,12 +723,12 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - tool-disclosure-smoke: + tool-disclosure-performance-smoke: needs: generate-matrix # This smoke provisions real agent sandboxes and consumes hosted inference. - # Keep it opt-in: it validates benchmark plumbing on ordinary CI hardware, - # but it is not the complete two-campaign claim-validation protocol. - if: ${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-smoke,') }} + # Keep it opt-in: it validates performance-test plumbing on ordinary CI hardware, + # but it is not the complete two-campaign performance-test procedure. + if: ${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-performance-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-performance-smoke,') }} runs-on: ubuntu-latest permissions: contents: read @@ -736,8 +736,8 @@ jobs: env: E2E_JOB: "1" E2E_DEFAULT_ENABLED: "0" - E2E_TARGET_ID: "tool-disclosure-smoke" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-smoke + E2E_TARGET_ID: "tool-disclosure-performance-smoke" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-performance-smoke NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" @@ -784,7 +784,7 @@ jobs: NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" run: bash scripts/install-openshell.sh - - name: Run tool-disclosure smoke live test + - name: Run tool-disclosure performance smoke live test env: NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} run: | @@ -801,10 +801,10 @@ jobs: export OPENSHELL_BIN "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ - test/e2e/live/tool-disclosure-smoke.test.ts \ + test/e2e/live/tool-disclosure-performance-smoke.test.ts \ --silent=false --reporter=default - - name: Upload tool-disclosure smoke artifacts + - name: Upload tool-disclosure performance smoke artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -4660,7 +4660,7 @@ jobs: openshell-gateway-auth-contract, mcp-bridge, mcp-bridge-dev, - tool-disclosure-smoke, + tool-disclosure-performance-smoke, onboard-negative-paths, skill-agent, openclaw-skill-cli, @@ -4765,10 +4765,10 @@ jobs: target: 'mcp-bridge-dev', reason: 'default dispatch excludes moving OpenShell dev artifacts unless explicitly selected', }, - 'tool-disclosure-smoke': { - job: 'tool-disclosure-smoke', - target: 'tool-disclosure-smoke', - reason: 'default dispatch excludes the hosted-inference benchmark smoke unless explicitly selected', + 'tool-disclosure-performance-smoke': { + job: 'tool-disclosure-performance-smoke', + target: 'tool-disclosure-performance-smoke', + reason: 'default dispatch excludes the hosted-inference performance smoke test unless explicitly selected', }, 'jetson-nvmap-gpu': { job: 'jetson-nvmap-gpu', @@ -4900,7 +4900,7 @@ jobs: ? '**Requested jobs:** _(selector rejected by workflow validation)_' : requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` - : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `tool-disclosure-smoke`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', + : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `tool-disclosure-performance-smoke`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`, '', '| Job | Result |', diff --git a/docs/index.yml b/docs/index.yml index cd4e206f0f..25a1525d97 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -66,9 +66,9 @@ navigation: - page: "Model Capability Audit" path: _build/agent-variants/inference/model-capability-audit.openclaw.generated.mdx slug: model-capability-audit - - page: "Progressive Tool Disclosure Benchmark" - path: _build/agent-variants/inference/progressive-tool-disclosure-benchmark.openclaw.generated.mdx - slug: progressive-tool-disclosure-benchmark + - page: "Progressive Tool Disclosure Performance Test" + path: _build/agent-variants/inference/progressive-tool-disclosure-performance-test.openclaw.generated.mdx + slug: progressive-tool-disclosure-performance-test - page: "Switch Inference Providers" path: _build/agent-variants/inference/switch-inference-providers.openclaw.generated.mdx slug: switch-inference-providers @@ -252,9 +252,9 @@ navigation: - page: "Model Capability Audit" path: _build/agent-variants/inference/model-capability-audit.hermes.generated.mdx slug: model-capability-audit - - page: "Progressive Tool Disclosure Benchmark" - path: _build/agent-variants/inference/progressive-tool-disclosure-benchmark.hermes.generated.mdx - slug: progressive-tool-disclosure-benchmark + - page: "Progressive Tool Disclosure Performance Test" + path: _build/agent-variants/inference/progressive-tool-disclosure-performance-test.hermes.generated.mdx + slug: progressive-tool-disclosure-performance-test - page: "Switch Inference Providers" path: _build/agent-variants/inference/switch-inference-providers.hermes.generated.mdx slug: switch-inference-providers diff --git a/docs/inference/progressive-tool-disclosure-benchmark.mdx b/docs/inference/progressive-tool-disclosure-performance-test.mdx similarity index 86% rename from docs/inference/progressive-tool-disclosure-benchmark.mdx rename to docs/inference/progressive-tool-disclosure-performance-test.mdx index c2f19086b1..45638718b3 100644 --- a/docs/inference/progressive-tool-disclosure-benchmark.mdx +++ b/docs/inference/progressive-tool-disclosure-performance-test.mdx @@ -1,11 +1,11 @@ --- # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Benchmark Progressive Tool Disclosure" -sidebar-title: "Progressive Tool Disclosure Benchmark" -description: "Run the reproducible two-campaign benchmark that measures progressive tool disclosure across NemoClaw agent surfaces." -description-agent: "Explains how to prepare, run, and summarize the hardware-neutral progressive tool-disclosure benchmark. Use when measuring tool-schema tokens, task success, or latency for OpenClaw, Hermes, and LangChain Deep Agents Code." -keywords: "progressive tool disclosure benchmark, hardware-neutral benchmark, tool schema tokens, MCP benchmark" +title: "Progressive Tool Disclosure Performance Test" +sidebar-title: "Progressive Tool Disclosure Performance Test" +description: "Run the reproducible two-campaign performance test that measures progressive tool disclosure across NemoClaw agent surfaces." +description-agent: "Explains how to prepare, run, and summarize the hardware-neutral progressive tool-disclosure performance test. Use when measuring tool-schema tokens, task success, or latency for OpenClaw, Hermes, and LangChain Deep Agents Code." +keywords: "progressive tool disclosure performance test, hardware-neutral performance test, tool schema tokens, MCP performance test" position: 5 --- @@ -17,9 +17,9 @@ The repository does not include measured results. Run both campaigns and retain their evidence before you make a performance or quality claim. -The explicit-only `tool-disclosure-smoke` E2E job is a smaller operational check for ordinary GitHub-hosted runners. +The explicit-only `tool-disclosure-performance-smoke` E2E job is a smaller operational check for ordinary GitHub-hosted runners. It runs one frozen task in progressive and direct modes with fresh Deep Agents sandboxes and hosted inference. -Its output validates live wiring only: it is not a complete campaign, cannot be summarized by this protocol, and does not support benchmark claims. +Its output validates live wiring only: it is not a complete campaign, cannot be summarized by this protocol, and does not support performance claims. ## Understand the frozen protocol @@ -44,7 +44,7 @@ The sandbox base must use an immutable OCI digest; mutable tags such as `latest` Place the evidence directory outside the Git worktree so generated artifacts cannot change the recorded cleanliness state. ```bash -npm run bench:tool-disclosure -- prepare \ +npm run performance:tool-disclosure -- prepare \ --output-dir \ --sandbox-base ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:<64-hex-digest> ``` @@ -68,18 +68,18 @@ For a CPU-only run, set `accelerator_count` to `0`, `accelerator_type` to `none` Keep credentials, endpoint URLs, hostnames, usernames, and local paths out of the manifest. -The benchmark commands do not provision hardware, start vLLM, or create sandboxes. +The performance test commands do not provision hardware, start vLLM, or create sandboxes. Prepare the runtime environment first; the `execute` command starts the loopback recorder and synthetic MCP fixtures, drives the frozen schedule, and writes sanitized raw evidence plus derived run records. -## Provision and preflight the benchmark host +## Provision and preflight the performance test host -In principle, the benchmark can run on any hardware with a compatible host environment that can serve the recorded model and complete the frozen schedule. +In principle, the performance test can run on any hardware with a compatible host environment that can serve the recorded model and complete the frozen schedule. The required runtime consists of Node.js and a POSIX-compatible shell, an OpenAI-compatible vLLM endpoint with `/metrics` and `/tokenize`, Docker-backed OpenShell sandboxes, `cloudflared` with outbound network access, and working OpenClaw, Hermes, and LangChain Deep Agents Code runtimes. -Provisioning and access procedures are outside the benchmark protocol. +Provisioning and access procedures are outside the performance test protocol. Run host-appropriate preflight checks before starting inference. -The following examples cover common Linux and macOS environments; run only the commands available on the benchmark host. +The following examples cover common Linux and macOS environments; run only the commands available on the performance test host. ```bash uname -a @@ -101,8 +101,8 @@ Do not publish raw preflight logs because they can contain hostnames, usernames, ## Route inference through the recorder -Run one vLLM server and one benchmark recording proxy on the benchmark host. -Configure the NemoClaw managed inference route so every model request from every benchmark sandbox passes through the proxy before reaching vLLM. +Run one vLLM server and one performance-test recording proxy on the performance test host. +Configure the NemoClaw managed inference route so every model request from every performance-test sandbox passes through the proxy before reaching vLLM. After vLLM starts, verify its local health endpoint before beginning a campaign. ```bash @@ -126,7 +126,7 @@ It does not retain prompts, messages, headers, request bodies, response bodies, Use vLLM `/tokenize` to count initial serialized tool-schema tokens with the exact served model tokenizer. Read prompt and generation token counters from vLLM `/metrics` immediately before and after each scheduled run. -Keep benchmark concurrency at `1` so those process-level counter deltas remain attributable to one run. +Keep performance-test concurrency at `1` so those process-level counter deltas remain attributable to one run. Use temperature `0`, the manifest's frozen parser configuration, and the same recorded vLLM flags in both modes and both campaigns. ## Run two fresh campaigns @@ -145,7 +145,7 @@ The quick-tunnel process disables config-file discovery and receives an isolated This attestation path expects vLLM and the OpenShell sandboxes to use the Docker driver so `docker inspect` can verify the live state. ```bash -npm run bench:tool-disclosure -- execute \ +npm run performance:tool-disclosure -- execute \ --output-dir \ --config ``` @@ -155,12 +155,12 @@ For campaign 1: 1. Start a fresh vLLM process from the recorded image digest and flags. 2. Start a fresh loopback recorder and reset its event stream. 3. Create fresh OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes for the scheduled modes and fixtures. -4. Run `execute` for campaign 1. It appends each attempt atomically to `attempt-journal.jsonl`, records hashed live identities in `campaign-attestations.json`, and materializes content-free `raw-events.jsonl` plus one derived `nemoclaw.tool_disclosure_bench.v1` row per entry in `runs.jsonl`. +4. Run `execute` for campaign 1. It appends each attempt atomically to `attempt-journal.jsonl`, records hashed live identities in `campaign-attestations.json`, and materializes content-free `raw-events.jsonl` plus one derived `nemoclaw.tool_disclosure_performance_test.v1` row per entry in `runs.jsonl`. 5. Stop vLLM, stop the recorder, and destroy the campaign sandboxes. For campaign 2, repeat the same five steps with another fresh vLLM process, recorder event stream, and sandbox set. Use the same hardware configuration, system-under-test and harness revisions, model revision, image digest, parser configuration, and task artifacts. -If any of those controlled values changes, prepare a new benchmark ID and restart both campaigns. +If any of those controlled values changes, prepare a new performance test ID and restart both campaigns. The agent adapters must use the generated deterministic handlers and verify expected tool names, order, argument hashes, call count, unnecessary calls, and the returned nonce. Direct and progressive modes must differ only in their disclosure configuration. @@ -172,7 +172,7 @@ Once an agent invocation has begun, any incomplete or inconsistent evidence abor After all scheduled records are present, run the summarizer in the same evidence directory. ```bash -npm run bench:tool-disclosure -- summarize --output-dir +npm run performance:tool-disclosure -- summarize --output-dir ``` The command consumes `manifest.json` and `runs.jsonl` and writes: @@ -185,7 +185,7 @@ The command consumes `manifest.json` and `runs.jsonl` and writes: - `evidence.json` with the public artifact inventory and SHA-256 values. - `SHA256SUMS` with the checksum manifest. -Verify the bundle on the benchmark host before you move it. +Verify the bundle on the performance test host before you move it. ```bash cd @@ -193,7 +193,7 @@ sha256sum -c SHA256SUMS # GNU/Linux shasum -a 256 -c SHA256SUMS # macOS ``` -Run the command available on the benchmark host. +Run the command available on the performance test host. Treat every output as untrusted until you review it for credentials, endpoint URLs, hostnames, usernames, local paths, raw prompts, tool arguments, and response text. Do not hand-edit `summary.json` or the validated claims in `report.md`. @@ -218,7 +218,7 @@ A blocked gate remains visible in `report.md`, and the report does not emit the ## Interpret the scope -The benchmark supports claims about the recorded commits, agents, model, serving configuration, hardware configuration, deterministic synthetic catalog, and task corpus. +The performance test supports claims about the recorded commits, agents, model, serving configuration, hardware configuration, deterministic synthetic catalog, and task corpus. It does not establish a universal result for other models, hardware, tool distributions, prompts, concurrency levels, or production workloads. Evaluate another hardware configuration with a separate complete two-campaign run and report that result with its own recorded environment metadata. diff --git a/package.json b/package.json index 1f38721bb5..0fc607c5fd 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "test:projects:check": "tsx scripts/checks/vitest-project-overlap.ts", "test:titles:check": "tsx scripts/checks/test-title-style.ts", "bench": "tsx scripts/bench/run.ts", - "bench:tool-disclosure": "tsx scripts/bench/tool-disclosure/run.ts", + "performance:tool-disclosure": "tsx scripts/performance/tool-disclosure/run.ts", "check": "npx prek run --all-files --stage pre-commit && npx prek run --all-files --stage manual", "check:diff": "npx prek run --from-ref origin/main --to-ref HEAD --stage pre-commit && npx commitlint --from origin/main --to HEAD && npx prek run --from-ref origin/main --to-ref HEAD --stage pre-push", "checks": "tsx scripts/checks/run.ts", diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 6fa7f99ca1..0977d9577d 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -118,62 +118,3 @@ compared without exposing sandbox names or credentials. The harness exits non-zero when a selected metric errors, a supplied trace is invalid, or required prerequisites (endpoint, model, API key) are missing. - -## Progressive tool-disclosure benchmark - -The progressive tool-disclosure benchmark compares direct and progressive tool -visibility for OpenClaw, Hermes, and LangChain Deep Agents Code against one -deterministic synthetic catalog and task corpus. -It is a claim-validation harness, not an extension of the advisory value -benchmark above. -In principle, it can run on any hardware with a compatible host environment -that supports the recorded model, Node.js and a POSIX-compatible shell, an -OpenAI-compatible vLLM endpoint with `/metrics` and `/tokenize`, Docker-backed -OpenShell sandboxes, `cloudflared` with outbound network access, and the three -agent runtimes. Results apply to the recorded hardware and configuration; -evaluating another hardware configuration requires a separate complete run. - -Prepare a new evidence directory with the frozen catalog, task sets, schedule, -manifest template, and generated OpenClaw fixtures: - -```bash -npm run bench:tool-disclosure -- prepare \ - --output-dir \ - --sandbox-base ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:<64-hex-digest> -``` - -Run the emitted schedule as two fresh campaigns on the same recorded hardware -and inference configuration. -Each campaign requires a fresh vLLM process and fresh sandboxes. -Populate `manifest.json` from `manifest.template.json`, and append the public-safe -run records to `runs.jsonl`. - -Drive each prepared campaign with a private configuration that identifies the -fresh vLLM container and maps the frozen agent/mode/catalog cells to fresh -sandbox names, container names, and instance IDs: - -```bash -npm run bench:tool-disclosure -- execute \ - --output-dir --config -``` - -Execution uses a recoverable `attempt-journal.jsonl`, verifies the live vLLM -image/configuration and Docker-backed sandbox identities, writes only hashed -private identities to the attestation artifact, materializes content-free -`raw-events.jsonl`, and derives `runs.jsonl`; the summarizer rejects records that -cannot be reproduced from the journal and raw evidence. - -Summarize the completed evidence set: - -```bash -npm run bench:tool-disclosure -- summarize --output-dir -``` - -The summarizer writes `summary.json`, `report.md`, `evidence.json`, and -`SHA256SUMS`. -It derives public claim text mechanically and leaves claims blocked unless the -required confidence-interval gate passes for every agent in both campaigns. - -See the [progressive tool-disclosure benchmark protocol](../../docs/inference/progressive-tool-disclosure-benchmark.mdx) -for the hardware-neutral workflow, recorder topology, artifact rules, claim -gates, and limitations. diff --git a/scripts/performance/README.md b/scripts/performance/README.md new file mode 100644 index 0000000000..e3a2be971f --- /dev/null +++ b/scripts/performance/README.md @@ -0,0 +1,62 @@ + + +# Progressive tool-disclosure performance test + +The progressive tool-disclosure performance test compares direct and +progressive tool visibility for OpenClaw, Hermes, and LangChain Deep Agents +Code against one deterministic synthetic catalog and task corpus. It is the +repeatable comparison harness for this feature. + +In principle, it can run on any hardware with a compatible host environment +that supports the recorded model, Node.js and a POSIX-compatible shell, an +OpenAI-compatible vLLM endpoint with `/metrics` and `/tokenize`, Docker-backed +OpenShell sandboxes, `cloudflared` with outbound network access, and the three +agent runtimes. Results apply to the recorded hardware and configuration; +evaluating another hardware configuration requires a separate complete run. + +Prepare a new evidence directory with the frozen catalog, task sets, schedule, +manifest template, and generated OpenClaw fixtures: + +```bash +npm run performance:tool-disclosure -- prepare \ + --output-dir \ + --sandbox-base ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:<64-hex-digest> +``` + +Run the emitted schedule as two fresh campaigns on the same recorded hardware +and inference configuration. Each campaign requires a fresh vLLM process and +fresh sandboxes. Populate `manifest.json` from `manifest.template.json`, and +append the public-safe run records to `runs.jsonl`. + +Drive each prepared campaign with a private configuration that identifies the +fresh vLLM container and maps the frozen agent/mode/catalog cells to fresh +sandbox names, container names, and instance IDs: + +```bash +npm run performance:tool-disclosure -- execute \ + --output-dir --config +``` + +Execution uses a recoverable `attempt-journal.jsonl`, verifies the live vLLM +image/configuration and Docker-backed sandbox identities, writes only hashed +private identities to the attestation artifact, materializes content-free +`raw-events.jsonl`, and derives `runs.jsonl`; the summarizer rejects records +that cannot be reproduced from the journal and raw evidence. + +Summarize the completed evidence set: + +```bash +npm run performance:tool-disclosure -- summarize --output-dir +``` + +The summarizer writes `summary.json`, `report.md`, `evidence.json`, and +`SHA256SUMS`. It derives public claim text mechanically and leaves claims +blocked unless the required confidence-interval gate passes for every agent in +both campaigns. + +See the [progressive tool-disclosure performance-test protocol](../../docs/inference/progressive-tool-disclosure-performance-test.mdx) +for the hardware-neutral workflow, recorder topology, artifact rules, claim +gates, and limitations. diff --git a/scripts/bench/tool-disclosure/artifacts.ts b/scripts/performance/tool-disclosure/artifacts.ts similarity index 86% rename from scripts/bench/tool-disclosure/artifacts.ts rename to scripts/performance/tool-disclosure/artifacts.ts index 097e4ff276..a3ed0bc4ce 100644 --- a/scripts/bench/tool-disclosure/artifacts.ts +++ b/scripts/performance/tool-disclosure/artifacts.ts @@ -12,11 +12,11 @@ export function ensureCampaignDirectory(outputDir: string, resume: boolean): str if (fs.existsSync(resolved)) { const stat = fs.lstatSync(resolved); if (stat.isSymbolicLink() || !stat.isDirectory()) { - throw new Error("benchmark output path must be a real directory"); + throw new Error("performance test output path must be a real directory"); } const entries = fs.readdirSync(resolved); if (!resume && entries.length > 0) { - throw new Error("benchmark output directory is not empty; pass --resume to reuse it"); + throw new Error("performance test output directory is not empty; pass --resume to reuse it"); } } else { fs.mkdirSync(resolved, { recursive: true, mode: 0o700 }); @@ -25,13 +25,14 @@ export function ensureCampaignDirectory(outputDir: string, resume: boolean): str } export function artifactPath(outputDir: string, name: string): string { - if (!SAFE_ARTIFACT_NAME.test(name)) throw new Error(`invalid benchmark artifact name: ${name}`); + if (!SAFE_ARTIFACT_NAME.test(name)) + throw new Error(`invalid performance test artifact name: ${name}`); const root = path.resolve(outputDir); const candidate = path.resolve(root, name); if (path.dirname(candidate) !== root) - throw new Error("benchmark artifact escaped output directory"); + throw new Error("performance test artifact escaped output directory"); if (fs.existsSync(candidate) && fs.lstatSync(candidate).isSymbolicLink()) { - throw new Error(`benchmark artifact must not be a symlink: ${name}`); + throw new Error(`performance test artifact must not be a symlink: ${name}`); } return candidate; } @@ -107,6 +108,6 @@ export function scanArtifactsForForbiddenValues( for (const name of names) { const content = fs.readFileSync(artifactPath(outputDir, name), "utf8"); const leaked = forbidden.find((value) => content.includes(value)); - if (leaked) throw new Error(`benchmark artifact ${name} contains a forbidden value`); + if (leaked) throw new Error(`performance test artifact ${name} contains a forbidden value`); } } diff --git a/scripts/bench/tool-disclosure/assemble-run.ts b/scripts/performance/tool-disclosure/assemble-run.ts similarity index 91% rename from scripts/bench/tool-disclosure/assemble-run.ts rename to scripts/performance/tool-disclosure/assemble-run.ts index 7a73b77516..c43c399e30 100644 --- a/scripts/bench/tool-disclosure/assemble-run.ts +++ b/scripts/performance/tool-disclosure/assemble-run.ts @@ -4,11 +4,11 @@ import { gradeTaskRun, type RecordedSyntheticCall } from "./grading"; import type { ToolDisclosureRecordingEvent } from "./recorder"; import type { ScheduledToolDisclosureRun } from "./schedule"; -import type { SyntheticBenchmarkTask } from "./tasks"; +import type { SyntheticPerformanceTask } from "./tasks"; import { - type BenchmarkRunOutcome, + type PerformanceTestRunOutcome, type RunCorrectness, - TOOL_DISCLOSURE_SCHEMA_VERSION, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, type ToolDisclosureManifest, type ToolDisclosureRun, } from "./types"; @@ -51,14 +51,14 @@ function failedCorrectness(callCount: number): RunCorrectness { export function assembleToolDisclosureRun(options: { manifest: ToolDisclosureManifest; scheduled: ScheduledToolDisclosureRun; - task?: SyntheticBenchmarkTask; + task?: SyntheticPerformanceTask; calls: readonly RecordedSyntheticCall[]; recorderEvents: readonly ToolDisclosureRecordingEvent[]; invocation: RunInvocationResult; initialSchemaTokens: number; promptTokens?: number; completionTokens?: number; - failureOutcome?: Exclude; + failureOutcome?: Exclude; }): ToolDisclosureRun { if (!Number.isSafeInteger(options.initialSchemaTokens) || options.initialSchemaTokens < 0) { throw new Error("initial schema token count must be a non-negative integer"); @@ -76,7 +76,7 @@ export function assembleToolDisclosureRun(options: { const modelEvents = options.recorderEvents.filter((event) => event.model_call_sequence !== null); const initial = modelEvents.find((event) => event.model_call_sequence === 1); - let outcome: BenchmarkRunOutcome; + let outcome: PerformanceTestRunOutcome; let correctness: RunCorrectness; if (options.failureOutcome) { outcome = options.failureOutcome; @@ -95,7 +95,7 @@ export function assembleToolDisclosureRun(options: { : failedCorrectness(options.calls.length); } else { const graded = gradeTaskRun( - options.task as SyntheticBenchmarkTask, + options.task as SyntheticPerformanceTask, options.calls, options.invocation.final_output, ); @@ -104,8 +104,8 @@ export function assembleToolDisclosureRun(options: { } return { - schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: options.manifest.benchmark_id, + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: options.manifest.performance_test_id, campaign_id: scheduledCampaign, run_id: options.scheduled.run_id, phase: options.scheduled.phase, diff --git a/scripts/bench/tool-disclosure/catalog.ts b/scripts/performance/tool-disclosure/catalog.ts similarity index 95% rename from scripts/bench/tool-disclosure/catalog.ts rename to scripts/performance/tool-disclosure/catalog.ts index 3c7f024457..f55429cca7 100644 --- a/scripts/bench/tool-disclosure/catalog.ts +++ b/scripts/performance/tool-disclosure/catalog.ts @@ -3,9 +3,10 @@ import { createHash } from "node:crypto"; -export const TOOL_DISCLOSURE_CATALOG_SCHEMA_VERSION = - "nemoclaw.tool_disclosure.catalog.v1" as const; -export const DEFAULT_SYNTHETIC_CATALOG_SEED = "nemoclaw-tool-disclosure-v1"; +export const TOOL_DISCLOSURE_PERFORMANCE_TEST_CATALOG_SCHEMA_VERSION = + "nemoclaw.tool_disclosure.performance_test.catalog.v1" as const; +export const DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED = + "nemoclaw-tool-disclosure-performance-test-v1"; export const MAX_SYNTHETIC_TOOLS = 2_209; export type JsonPrimitive = string | number | boolean | null; @@ -240,7 +241,7 @@ export interface SyntheticTool { } export interface SyntheticCatalog { - schema_version: typeof TOOL_DISCLOSURE_CATALOG_SCHEMA_VERSION; + schema_version: typeof TOOL_DISCLOSURE_PERFORMANCE_TEST_CATALOG_SCHEMA_VERSION; seed: string; size: number; max_size: typeof MAX_SYNTHETIC_TOOLS; @@ -268,7 +269,7 @@ export class SyntheticToolInvocationError extends Error {} const TOOL_NAME_PATTERN = /^[A-Za-z0-9_-]{1,64}$/; // A 25/50/25 cycle makes the medium schema the catalog median while retaining // substantial small and large populations for scaling comparisons. The medium -// definition is intentionally near the benchmark's roughly 400-token target; +// definition is intentionally near the performance test's roughly 400-token target; // authoritative token counts still come from the pinned model tokenizer. const COMPLEXITY_CYCLE: readonly SchemaComplexity[] = [ "small", @@ -462,7 +463,7 @@ function buildParameters( return { type: "object", - description: `Inputs for a synthetic ${category.label} benchmark operation.`, + description: `Inputs for a synthetic ${category.label} performance test operation.`, properties, required, additionalProperties: false, @@ -481,7 +482,7 @@ function generateTool( const operation = category.operations[(occurrence + operationOffset) % category.operations.length]; const complexity = complexityForIndex(index); - const name = `bench_${category.id}_${operation}_${index.toString().padStart(4, "0")}`; + const name = `performance_test_${category.id}_${operation}_${index.toString().padStart(4, "0")}`; if (!TOOL_NAME_PATTERN.test(name)) throw new Error(`generated unsafe tool name: ${name}`); @@ -496,7 +497,7 @@ function generateTool( name, description: [ `Synthetic ${category.label} fixture: ${operation.replaceAll("_", " ")} one ${category.resource}.`, - `Use for benchmark requests about ${category.purpose}; it never contacts a real service.`, + `Use for performance test requests about ${category.purpose}; it never contacts a real service.`, ].join(" "), parameters: buildParameters(category, operation, complexity), }, @@ -516,7 +517,7 @@ function assertCatalogSize(size: number): void { export function generateSyntheticTools( size = MAX_SYNTHETIC_TOOLS, - seed: string | number = DEFAULT_SYNTHETIC_CATALOG_SEED, + seed: string | number = DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, ): SyntheticTool[] { assertCatalogSize(size); const normalizedSeed = normalizeSeed(seed); @@ -528,10 +529,10 @@ export function generateSyntheticTools( export function generateSyntheticCatalog(options: SyntheticCatalogOptions = {}): SyntheticCatalog { const size = options.size ?? MAX_SYNTHETIC_TOOLS; - const seed = normalizeSeed(options.seed ?? DEFAULT_SYNTHETIC_CATALOG_SEED); + const seed = normalizeSeed(options.seed ?? DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED); const tools = generateSyntheticTools(size, seed); return { - schema_version: TOOL_DISCLOSURE_CATALOG_SCHEMA_VERSION, + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_CATALOG_SCHEMA_VERSION, seed, size, max_size: MAX_SYNTHETIC_TOOLS, @@ -699,7 +700,7 @@ export function executeSyntheticTool(tool: SyntheticTool, args: JsonObject): Syn export function validateSyntheticCatalog(catalog: SyntheticCatalog): string[] { const errors: string[] = []; - if (catalog.schema_version !== TOOL_DISCLOSURE_CATALOG_SCHEMA_VERSION) { + if (catalog.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_CATALOG_SCHEMA_VERSION) { errors.push(`unsupported catalog schema version: ${catalog.schema_version}`); } if (catalog.size !== catalog.tools.length) errors.push("catalog size does not match tool count"); diff --git a/scripts/bench/tool-disclosure/drivers.ts b/scripts/performance/tool-disclosure/drivers.ts similarity index 94% rename from scripts/bench/tool-disclosure/drivers.ts rename to scripts/performance/tool-disclosure/drivers.ts index a05ed996d5..518910b7db 100644 --- a/scripts/bench/tool-disclosure/drivers.ts +++ b/scripts/performance/tool-disclosure/drivers.ts @@ -11,7 +11,8 @@ export interface AgentDriverCommand { redactions: string[]; } -export const OPENCLAW_BENCH_CALLS_PATH = "/sandbox/.nemoclaw-benchmark/calls.jsonl"; +export const OPENCLAW_PERFORMANCE_TEST_CALLS_PATH = + "/sandbox/.nemoclaw-performance-test/calls.jsonl"; function validateIdentifier(value: string, label: string): void { if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/u.test(value)) { @@ -70,7 +71,7 @@ export function buildAgentDriverCommand(options: { }): AgentDriverCommand { validateIdentifier(options.sandboxName, "sandboxName"); validateIdentifier(options.sessionId, "sessionId"); - if (!options.prompt.trim()) throw new Error("benchmark task prompt must not be empty"); + if (!options.prompt.trim()) throw new Error("performance test task prompt must not be empty"); const script = shellScriptFor(options.agent, options.prompt, options.sessionId); return { command: options.openshellBin ?? "openshell", @@ -186,8 +187,8 @@ export function buildOpenClawCallLogCommand(options: { validateIdentifier(options.sandboxName, "sandboxName"); const script = options.action === "reset" - ? `rm -f -- '${OPENCLAW_BENCH_CALLS_PATH}'` - : `[ ! -f '${OPENCLAW_BENCH_CALLS_PATH}' ] || cat -- '${OPENCLAW_BENCH_CALLS_PATH}'`; + ? `rm -f -- '${OPENCLAW_PERFORMANCE_TEST_CALLS_PATH}'` + : `[ ! -f '${OPENCLAW_PERFORMANCE_TEST_CALLS_PATH}' ] || cat -- '${OPENCLAW_PERFORMANCE_TEST_CALLS_PATH}'`; return { command: options.openshellBin ?? "openshell", args: ["sandbox", "exec", "-n", options.sandboxName, "--", "sh", "-lc", script], diff --git a/scripts/bench/tool-disclosure/execute.ts b/scripts/performance/tool-disclosure/execute.ts similarity index 97% rename from scripts/bench/tool-disclosure/execute.ts rename to scripts/performance/tool-disclosure/execute.ts index 7367117e31..ca84d4716f 100644 --- a/scripts/bench/tool-disclosure/execute.ts +++ b/scripts/performance/tool-disclosure/execute.ts @@ -18,7 +18,7 @@ import { type QuickTunnel, startQuickTunnel } from "./quick-tunnel"; import { createToolDisclosureRecordingProxy, type ToolDisclosureRecordingEvent } from "./recorder"; import type { ScheduledToolDisclosureRun, ToolDisclosureAgent } from "./schedule"; import { runBoundedCommand } from "./subprocess"; -import type { SyntheticBenchmarkTask, SyntheticTaskSet } from "./tasks"; +import type { SyntheticPerformanceTask, SyntheticTaskSet } from "./tasks"; import { countTokensWithVllm, readVllmProcessStartTime, @@ -558,7 +558,7 @@ async function configureMcpSandbox(options: { "--timeout", "15m", "--reason", - "tool disclosure benchmark setup", + "tool disclosure performance test setup", ]); } try { @@ -566,14 +566,14 @@ async function configureMcpSandbox(options: { if (listed.exit_code !== 0 || listed.timed_out || listed.output_truncated) { throw new Error(`MCP inventory failed for ${options.agent} sandbox`); } - if (/"(?:name|server)"\s*:\s*"benchmark"/u.test(listed.stdout)) { - await run([options.sandboxName, "mcp", "remove", "benchmark", "--force"]); + if (/"(?:name|server)"\s*:\s*"performance-test"/u.test(listed.stdout)) { + await run([options.sandboxName, "mcp", "remove", "performance-test", "--force"]); } await run([ options.sandboxName, "mcp", "add", - "benchmark", + "performance-test", "--url", options.url, "--env", @@ -587,7 +587,7 @@ async function configureMcpSandbox(options: { function taskMap( primary: SyntheticTaskSet, stress: SyntheticTaskSet, -): Map { +): Map { return new Map([...primary.tasks, ...stress.tasks].map((task) => [task.id, task] as const)); } @@ -612,10 +612,10 @@ export async function executeCampaign(options: { } const names = [...expectedKeys].map((key) => options.config.sandbox_names[key]); if (new Set(names).size !== names.length) - throw new Error("every benchmark cell needs a unique sandbox"); + throw new Error("every performance test cell needs a unique sandbox"); const instanceIds = [...expectedKeys].map((key) => options.config.sandbox_instance_ids[key]); if (instanceIds.some((value) => !value) || new Set(instanceIds).size !== instanceIds.length) { - throw new Error("every benchmark cell needs a unique live sandbox instance ID"); + throw new Error("every performance test cell needs a unique live sandbox instance ID"); } const containerNames = [...expectedKeys].map( (key) => options.config.sandbox_container_names[key], @@ -624,7 +624,7 @@ export async function executeCampaign(options: { containerNames.some((value) => !value) || new Set(containerNames).size !== containerNames.length ) { - throw new Error("every benchmark cell needs a unique live container name"); + throw new Error("every performance test cell needs a unique live container name"); } if (options.config.tokenizer_model !== options.manifest.inference.model_id) { throw new Error("tokenizer_model must exactly match manifest.inference.model_id"); @@ -641,7 +641,7 @@ export async function executeCampaign(options: { existing.filter((run) => run.outcome !== "setup-error").map((run) => run.run_id), ); const token = randomBytes(32).toString("hex"); - const tokenEnv = "NEMOCLAW_BENCHMARK_MCP_BEARER"; + const tokenEnv = "NEMOCLAW_PERFORMANCE_TEST_MCP_BEARER"; const proxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl: options.config.upstream_vllm_url, port: options.config.recorder_port, diff --git a/scripts/bench/tool-disclosure/grading.ts b/scripts/performance/tool-disclosure/grading.ts similarity index 90% rename from scripts/bench/tool-disclosure/grading.ts rename to scripts/performance/tool-disclosure/grading.ts index fad0b04c1e..7a06a990a2 100644 --- a/scripts/bench/tool-disclosure/grading.ts +++ b/scripts/performance/tool-disclosure/grading.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { canonicalJson, sha256Hex } from "./catalog"; -import type { SyntheticBenchmarkTask } from "./tasks"; -import type { BenchmarkRunOutcome, RunCorrectness } from "./types"; +import type { SyntheticPerformanceTask } from "./tasks"; +import type { PerformanceTestRunOutcome, RunCorrectness } from "./types"; export interface RecordedSyntheticCall { tool_name: string; @@ -13,13 +13,13 @@ export interface RecordedSyntheticCall { } export interface GradedTaskRun { - outcome: Extract; + outcome: Extract; correctness: RunCorrectness; } /** Grade a run only from deterministic call metadata and the final model output. */ export function gradeTaskRun( - task: SyntheticBenchmarkTask, + task: SyntheticPerformanceTask, calls: readonly RecordedSyntheticCall[], finalOutput: string, ): GradedTaskRun { diff --git a/scripts/bench/tool-disclosure/mcp-server.ts b/scripts/performance/tool-disclosure/mcp-server.ts similarity index 89% rename from scripts/bench/tool-disclosure/mcp-server.ts rename to scripts/performance/tool-disclosure/mcp-server.ts index 14b486f4ba..6519377140 100644 --- a/scripts/bench/tool-disclosure/mcp-server.ts +++ b/scripts/performance/tool-disclosure/mcp-server.ts @@ -101,18 +101,18 @@ export class SyntheticMcpServer { constructor(catalog: SyntheticCatalog, bearerToken: string) { if (!bearerToken || bearerToken.length > 4_096) { - throw new Error("benchmark MCP bearer token must be non-empty and bounded"); + throw new Error("performance test MCP bearer token must be non-empty and bounded"); } this.bearerToken = bearerToken; this.toolsByName = new Map( catalog.tools.map((tool) => [tool.definition.function.name, tool] as const), ); if (this.toolsByName.size !== catalog.size) { - throw new Error("benchmark MCP catalog contains duplicate tool names"); + throw new Error("performance test MCP catalog contains duplicate tool names"); } this.server = http.createServer((request, response) => { void this.handle(request, response).catch(() => { - jsonResponse(response, 500, { error: "benchmark MCP server failure" }); + jsonResponse(response, 500, { error: "performance test MCP server failure" }); }); }); this.server.requestTimeout = 120_000; @@ -121,7 +121,7 @@ export class SyntheticMcpServer { async start(port = 0): Promise { if (!Number.isInteger(port) || port < 0 || port > 65_535) { - throw new Error("benchmark MCP port must be between 0 and 65535"); + throw new Error("performance test MCP port must be between 0 and 65535"); } await new Promise((resolve, reject) => { this.server.once("error", reject); @@ -130,7 +130,7 @@ export class SyntheticMcpServer { const address = this.server.address() as AddressInfo | null; if (!address || address.address !== "127.0.0.1") { await this.stop(); - throw new Error("benchmark MCP server failed to bind loopback"); + throw new Error("performance test MCP server failed to bind loopback"); } return { host: "127.0.0.1", @@ -144,14 +144,14 @@ export class SyntheticMcpServer { } beginRun(runId: string): void { - if (!SAFE_RUN_ID.test(runId)) throw new Error("invalid benchmark MCP run id"); - if (this.activeRun) throw new Error("a benchmark MCP run is already active"); + if (!SAFE_RUN_ID.test(runId)) throw new Error("invalid performance test MCP run id"); + if (this.activeRun) throw new Error("a performance test MCP run is already active"); this.activeRun = { id: runId, sequence: 0, eventStartIndex: this.events.length }; } endRun(): SyntheticMcpCallEvent[] { const run = this.activeRun; - if (!run) throw new Error("no benchmark MCP run is active"); + if (!run) throw new Error("no performance test MCP run is active"); this.activeRun = null; return this.events .slice(run.eventStartIndex) @@ -228,7 +228,7 @@ export class SyntheticMcpServer { protocolVersion: typeof params.protocolVersion === "string" ? params.protocolVersion : "2025-03-26", capabilities: { tools: {} }, - serverInfo: { name: "nemoclaw-tool-disclosure-benchmark", version: "1.0.0" }, + serverInfo: { name: "nemoclaw-tool-disclosure-performance-test", version: "1.0.0" }, }, }); return; @@ -251,7 +251,7 @@ export class SyntheticMcpServer { jsonResponse(response, 200, { jsonrpc: "2.0", id, - error: { code: -32602, message: "invalid benchmark tool call" }, + error: { code: -32602, message: "invalid performance test tool call" }, }); return; } @@ -268,7 +268,7 @@ export class SyntheticMcpServer { jsonResponse(response, 200, { jsonrpc: "2.0", id, - error: { code: -32602, message: "invalid benchmark tool arguments" }, + error: { code: -32602, message: "invalid performance test tool arguments" }, }); } return; diff --git a/scripts/bench/tool-disclosure/openclaw-fixture.ts b/scripts/performance/tool-disclosure/openclaw-fixture.ts similarity index 90% rename from scripts/bench/tool-disclosure/openclaw-fixture.ts rename to scripts/performance/tool-disclosure/openclaw-fixture.ts index 72d1d49f39..07ac22d0e6 100644 --- a/scripts/bench/tool-disclosure/openclaw-fixture.ts +++ b/scripts/performance/tool-disclosure/openclaw-fixture.ts @@ -39,12 +39,12 @@ export function assertImmutableSandboxBase(image: string): void { } function pluginSource(): string { - return `// Generated by the NemoClaw progressive tool-disclosure benchmark.\n\ + return `// Generated by the NemoClaw progressive tool-disclosure performance test.\n\ import { createHash } from "node:crypto";\n\ import fs from "node:fs";\n\ \n\ const catalog = JSON.parse(fs.readFileSync(new URL("./catalog.json", import.meta.url), "utf8"));\n\ -const callsPath = process.env.NEMOCLAW_BENCH_CALLS_PATH || "/sandbox/.nemoclaw-benchmark/calls.jsonl";\n\ +const callsPath = process.env.NEMOCLAW_PERFORMANCE_TEST_CALLS_PATH || "/sandbox/.nemoclaw-performance-test/calls.jsonl";\n\ \n\ function canonical(value) {\n\ if (Array.isArray(value)) return "[" + value.map(canonical).join(",") + "]";\n\ @@ -79,7 +79,7 @@ function execute(tool, args) {\n\ return result;\n\ }\n\ \n\ -export default function registerBenchmarkTools(api) {\n\ +export default function registerPerformanceTestTools(api) {\n\ if (api.registrationMode && api.registrationMode !== "full") return;\n\ for (const tool of catalog.tools) {\n\ const definition = tool.definition.function;\n\ @@ -107,8 +107,8 @@ FROM \${SANDBOX_BASE}\n\ ARG NEMOCLAW_TOOL_DISCLOSURE=progressive\n\ ENV NEMOCLAW_TOOL_DISCLOSURE=\${NEMOCLAW_TOOL_DISCLOSURE}\n\ \n\ -COPY plugin/ /opt/nemoclaw-tool-disclosure-benchmark/\n\ -RUN mkdir -p /sandbox/.openclaw/extensions \\\n && cp -a /opt/nemoclaw-tool-disclosure-benchmark /sandbox/.openclaw/extensions/nemoclaw-tool-disclosure-benchmark \\\n && install -d -m 2770 -o sandbox -g sandbox /sandbox/.nemoclaw-benchmark \\\n && openclaw doctor --fix\n\ +COPY plugin/ /opt/nemoclaw-tool-disclosure-performance-test/\n\ +RUN mkdir -p /sandbox/.openclaw/extensions \\\n && cp -a /opt/nemoclaw-tool-disclosure-performance-test /sandbox/.openclaw/extensions/nemoclaw-tool-disclosure-performance-test \\\n && install -d -m 2770 -o sandbox -g sandbox /sandbox/.nemoclaw-performance-test \\\n && openclaw doctor --fix\n\ \n\ WORKDIR /opt/nemoclaw\n`; } @@ -151,7 +151,7 @@ export function writeOpenClawFixture(options: OpenClawFixtureOptions): OpenClawF "package.json", `${JSON.stringify( { - name: "nemoclaw-tool-disclosure-benchmark", + name: "nemoclaw-tool-disclosure-performance-test", version: "1.0.0", private: true, type: "module", @@ -166,10 +166,10 @@ export function writeOpenClawFixture(options: OpenClawFixtureOptions): OpenClawF "openclaw.plugin.json", `${JSON.stringify( { - id: "nemoclaw-tool-disclosure-benchmark", - name: "NemoClaw Tool Disclosure Benchmark", + id: "nemoclaw-tool-disclosure-performance-test", + name: "NemoClaw Tool Disclosure Performance Test", version: "1.0.0", - description: "Deterministic synthetic tools for progressive disclosure benchmarks.", + description: "Deterministic synthetic tools for progressive disclosure performance tests.", contracts: { tools: names }, configSchema: { type: "object", properties: {}, additionalProperties: false }, }, diff --git a/scripts/bench/tool-disclosure/quick-tunnel.ts b/scripts/performance/tool-disclosure/quick-tunnel.ts similarity index 100% rename from scripts/bench/tool-disclosure/quick-tunnel.ts rename to scripts/performance/tool-disclosure/quick-tunnel.ts diff --git a/scripts/bench/tool-disclosure/recorder.ts b/scripts/performance/tool-disclosure/recorder.ts similarity index 98% rename from scripts/bench/tool-disclosure/recorder.ts rename to scripts/performance/tool-disclosure/recorder.ts index 93f856f1f9..189f809c70 100644 --- a/scripts/bench/tool-disclosure/recorder.ts +++ b/scripts/performance/tool-disclosure/recorder.ts @@ -67,7 +67,7 @@ export type RecorderErrorReason = * * Do not add URLs, headers, request/response bodies, model identifiers, tool * descriptions, or thrown error messages to this interface. This object is - * suitable for direct inclusion in the public benchmark evidence bundle. + * suitable for direct inclusion in the public performance test evidence bundle. */ export interface ToolDisclosureRecordingEvent { run_id: string; @@ -375,7 +375,7 @@ function readBoundedBody(request: IncomingMessage, maxBytes: number): Promise { void this.handleRequest(request, response).catch(() => { // Keep thrown implementation errors content-free at the network and - // evidence boundaries. The benchmark runner receives a fixed class. + // evidence boundaries. The performance test runner receives a fixed class. fixedResponse(response, 502, "recording proxy failure"); }); }); @@ -696,7 +696,7 @@ export class ToolDisclosureRecordingProxy { pending.metrics.temperature !== null && pending.metrics.temperature !== this.requiredTemperature ) { - fixedResponse(response, 400, "benchmark temperature mismatch"); + fixedResponse(response, 400, "performance test temperature mismatch"); this.finalizeRecording(pending, { statusCode: 400, outcome: "request-rejected", diff --git a/scripts/bench/tool-disclosure/report.ts b/scripts/performance/tool-disclosure/report.ts similarity index 94% rename from scripts/bench/tool-disclosure/report.ts rename to scripts/performance/tool-disclosure/report.ts index f607673858..3fb46cd399 100644 --- a/scripts/bench/tool-disclosure/report.ts +++ b/scripts/performance/tool-disclosure/report.ts @@ -5,7 +5,7 @@ import { buildConservativeClaims } from "./statistics"; import { type ConfidenceInterval, type EvidenceArtifact, - TOOL_DISCLOSURE_SCHEMA_VERSION, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, type ToolDisclosureManifest, type ToolDisclosureSummary, } from "./types"; @@ -30,10 +30,10 @@ export function renderToolDisclosureMarkdown( } const lines: string[] = [ - "# Progressive Tool-Disclosure Benchmark", + "# Progressive Tool-Disclosure Performance Test", "", - `Schema: \`${TOOL_DISCLOSURE_SCHEMA_VERSION}\` `, - `Benchmark: \`${safeInline(manifest.benchmark_id)}\` `, + `Schema: \`${TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION}\` `, + `Performance Test: \`${safeInline(manifest.performance_test_id)}\` `, `Generated: ${safeInline(summary.generated_at)} `, `SUT: \`${safeInline(manifest.sut.git_ref)}\` at \`${safeInline(manifest.sut.git_sha)}\` `, `Harness: \`${safeInline(manifest.harness.git_ref)}\` at \`${safeInline( @@ -205,13 +205,13 @@ function validateInputs( ): void { validatePublicManifestFields(manifest); if ( - manifest.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION || - summary.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION + manifest.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION || + summary.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION ) { - throw new Error("manifest and summary must use the tool-disclosure benchmark schema"); + throw new Error("manifest and summary must use the tool-disclosure performance test schema"); } - if (manifest.benchmark_id !== summary.benchmark_id) { - throw new Error("manifest and summary benchmark IDs do not match"); + if (manifest.performance_test_id !== summary.performance_test_id) { + throw new Error("manifest and summary performance test IDs do not match"); } for (const artifact of artifacts) { if (artifact.file_name.includes("/") || artifact.file_name.includes("\\")) { diff --git a/scripts/bench/tool-disclosure/run.ts b/scripts/performance/tool-disclosure/run.ts similarity index 94% rename from scripts/bench/tool-disclosure/run.ts rename to scripts/performance/tool-disclosure/run.ts index d842392245..7e0570c735 100644 --- a/scripts/bench/tool-disclosure/run.ts +++ b/scripts/performance/tool-disclosure/run.ts @@ -18,7 +18,7 @@ import { } from "./artifacts"; import { assertValidSyntheticCatalog, - DEFAULT_SYNTHETIC_CATALOG_SEED, + DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, generateCatalogPrefix, generateSyntheticCatalog, type SyntheticCatalog, @@ -51,7 +51,7 @@ import { DEFAULT_NONINFERIORITY_MARGIN_PP, type EvidenceArtifact, type EvidenceBundle, - TOOL_DISCLOSURE_SCHEMA_VERSION, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, type ToolDisclosureManifest, type ToolDisclosureRun, } from "./types"; @@ -86,11 +86,11 @@ const PREPARED_FILES = [ ] as const; function usage(): string { - return `Progressive tool-disclosure benchmark\n\n\ + return `Progressive tool-disclosure performance test\n\n\ Usage:\n\ - npm run bench:tool-disclosure -- prepare --output-dir [options]\n\ - npm run bench:tool-disclosure -- execute --output-dir --config \n\ - npm run bench:tool-disclosure -- summarize --output-dir [options]\n\n\ + npm run performance:tool-disclosure -- prepare --output-dir [options]\n\ + npm run performance:tool-disclosure -- execute --output-dir --config \n\ + npm run performance:tool-disclosure -- summarize --output-dir [options]\n\n\ Prepare options:\n\ --sut-ref SUT revision to resolve (default: HEAD)\n\ --catalog-seed Deterministic catalog seed\n\ @@ -119,7 +119,7 @@ function parseArguments(args: readonly string[]): ParsedArguments { const parsed: ParsedArguments = { command, sutRef: "HEAD", - catalogSeed: DEFAULT_SYNTHETIC_CATALOG_SEED, + catalogSeed: DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, executionSeed: 0x4e_56_44_41, resume: false, allowDirty: false, @@ -171,7 +171,7 @@ function revision(ref: string, worktreeClean: boolean): ToolDisclosureManifest[" return { git_sha: sha, git_ref: ref, worktree_clean: worktreeClean }; } -function benchmarkId(sha: string, catalogHash: string): string { +function performanceTestId(sha: string, catalogHash: string): string { return `tool-disclosure-${sha.slice(0, 12)}-${catalogHash.slice(0, 12)}`; } @@ -188,8 +188,8 @@ function manifestTemplate(options: { const sut = revision(options.sutRef, options.worktreeClean); const harness = revision("HEAD", options.worktreeClean); return { - schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: benchmarkId(sut.git_sha, options.catalogHash), + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: performanceTestId(sut.git_sha, options.catalogHash), created_at: new Date().toISOString(), sut, harness, @@ -277,12 +277,12 @@ function prepare(options: ParsedArguments): void { requestedOutput === repositoryRoot || requestedOutput.startsWith(`${repositoryRoot}${path.sep}`) ) { - throw new Error("benchmark output directory must be outside the Git worktree"); + throw new Error("performance test output directory must be outside the Git worktree"); } const worktreeClean = gitOutput(["status", "--porcelain=v1", "--untracked-files=normal"]) === ""; if (!worktreeClean && !options.allowDirty) { throw new Error( - "benchmark preparation requires a clean worktree; commit or stash changes first", + "performance test preparation requires a clean worktree; commit or stash changes first", ); } if (!options.sandboxBase) { @@ -443,8 +443,10 @@ function assertPreparedIdentity( primaryTasks: SyntheticTaskSet, stressTasks: SyntheticTaskSet, ): void { - if (manifest.benchmark_id !== benchmarkId(manifest.sut.git_sha, catalog.tools_sha256)) { - throw new Error("manifest benchmark ID does not match the frozen SUT and catalog"); + if ( + manifest.performance_test_id !== performanceTestId(manifest.sut.git_sha, catalog.tools_sha256) + ) { + throw new Error("manifest performance test ID does not match the frozen SUT and catalog"); } const expectedTasks = [...primaryTasks.tasks, ...stressTasks.tasks].map((task) => ({ task_id: task.id, @@ -635,8 +637,8 @@ function summarize(options: ParsedArguments): void { ); const artifacts = [...reportInputs, evidenceArtifact(outputDir, "report.md", "report")]; const bundle: EvidenceBundle = { - schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: manifest.benchmark_id, + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: manifest.performance_test_id, generated_at: new Date().toISOString(), artifacts, }; @@ -717,7 +719,7 @@ export async function main(args = process.argv.slice(2)): Promise { if (import.meta.url === `file://${process.argv[1]}`) { void main().catch((error: unknown) => { process.stderr.write( - `tool-disclosure benchmark: ${error instanceof Error ? error.message : String(error)}\n`, + `tool-disclosure performance test: ${error instanceof Error ? error.message : String(error)}\n`, ); process.exitCode = 1; }); diff --git a/scripts/bench/tool-disclosure/schedule.ts b/scripts/performance/tool-disclosure/schedule.ts similarity index 100% rename from scripts/bench/tool-disclosure/schedule.ts rename to scripts/performance/tool-disclosure/schedule.ts diff --git a/scripts/bench/tool-disclosure/statistics.ts b/scripts/performance/tool-disclosure/statistics.ts similarity index 97% rename from scripts/bench/tool-disclosure/statistics.ts rename to scripts/performance/tool-disclosure/statistics.ts index 0239725e76..fd0d637abf 100644 --- a/scripts/bench/tool-disclosure/statistics.ts +++ b/scripts/performance/tool-disclosure/statistics.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { - type BenchmarkPhase, + type PerformanceTestPhase, type CampaignAgentGate, type ClaimGateSummary, type ComparisonCellSummary, @@ -13,7 +13,7 @@ import { type ModeAggregate, type SchemaReductionSummary, type SchemaVisibilitySnapshot, - TOOL_DISCLOSURE_SCHEMA_VERSION, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, type ToolDisclosureAgent, type ToolDisclosureManifest, type ToolDisclosureMode, @@ -50,7 +50,7 @@ interface TaskAggregate { interface ComparisonGroup { campaignId: string; - phase: Exclude; + phase: Exclude; agent: ToolDisclosureAgent; catalogSize: number; runs: ToolDisclosureRun[]; @@ -63,7 +63,7 @@ interface StaticGroup { runs: ToolDisclosureRun[]; } -const PHASE_ORDER: Readonly, number>> = { +const PHASE_ORDER: Readonly, number>> = { "small-control": 0, primary: 1, "large-stress": 2, @@ -330,8 +330,8 @@ export function buildToolDisclosureSummary( noninferiorityMarginPercentagePoints: manifest.protocol.noninferiority_margin_percentage_points, }); const summaryWithoutClaims: ToolDisclosureSummary = { - schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: manifest.benchmark_id, + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: manifest.performance_test_id, generated_at: options.generatedAt ?? new Date().toISOString(), bootstrap_samples: manifest.protocol.bootstrap_samples, bootstrap_seed: normalizeSeed(manifest.protocol.bootstrap_seed), @@ -598,7 +598,7 @@ function validateEvidenceSet( manifest: ToolDisclosureManifest, runs: readonly ToolDisclosureRun[], ): void { - if (manifest.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION) { + if (manifest.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION) { throw new Error(`unsupported manifest schema: ${manifest.schema_version}`); } if (manifest.campaigns.length !== 2) { @@ -608,14 +608,14 @@ function validateEvidenceSet( if (run.schema_version !== manifest.schema_version) { throw new Error(`run ${run.run_id} uses a different schema version`); } - if (run.benchmark_id !== manifest.benchmark_id) { - throw new Error(`run ${run.run_id} belongs to a different benchmark`); + if (run.performance_test_id !== manifest.performance_test_id) { + throw new Error(`run ${run.run_id} belongs to a different performance test`); } } } function validateRun(run: ToolDisclosureRun): void { - if (run.schema_version !== TOOL_DISCLOSURE_SCHEMA_VERSION) { + if (run.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION) { throw new Error(`run ${run.run_id} has unsupported schema ${run.schema_version}`); } const shouldBeScored = run.phase !== "static-visibility" && run.outcome !== "setup-error"; diff --git a/scripts/bench/tool-disclosure/subprocess.ts b/scripts/performance/tool-disclosure/subprocess.ts similarity index 95% rename from scripts/bench/tool-disclosure/subprocess.ts rename to scripts/performance/tool-disclosure/subprocess.ts index 291967c728..c2bc1c3978 100644 --- a/scripts/bench/tool-disclosure/subprocess.ts +++ b/scripts/performance/tool-disclosure/subprocess.ts @@ -32,9 +32,9 @@ export async function runBoundedCommand(options: { redactions?: readonly string[]; }): Promise { if (!options.command || options.command.includes("\0")) - throw new Error("invalid benchmark command"); + throw new Error("invalid performance test command"); if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1) { - throw new Error("benchmark command timeout must be a positive integer"); + throw new Error("performance test command timeout must be a positive integer"); } const maxOutputBytes = options.maxOutputBytes ?? 2 * 1024 * 1024; const child = spawn(options.command, [...options.args], { diff --git a/scripts/bench/tool-disclosure/tasks.ts b/scripts/performance/tool-disclosure/tasks.ts similarity index 90% rename from scripts/bench/tool-disclosure/tasks.ts rename to scripts/performance/tool-disclosure/tasks.ts index 4a7d86625b..97d6b7238c 100644 --- a/scripts/bench/tool-disclosure/tasks.ts +++ b/scripts/performance/tool-disclosure/tasks.ts @@ -14,14 +14,15 @@ import { validateSyntheticArguments, } from "./catalog"; -export const TOOL_DISCLOSURE_TASK_SCHEMA_VERSION = "nemoclaw.tool_disclosure.tasks.v1" as const; +export const TOOL_DISCLOSURE_PERFORMANCE_TEST_TASK_SCHEMA_VERSION = + "nemoclaw.tool_disclosure.performance_test.tasks.v1" as const; export const PRIMARY_TASK_COUNT = 24; export const PRIMARY_CATALOG_MIN_SIZE = 64; export const STRESS_TASK_COUNT = 8; export const STRESS_CATALOG_SIZE = 2_209; -export type BenchmarkTaskKind = "single-tool" | "ordered-chain" | "near-match" | "no-tool"; -export type BenchmarkTaskSuite = "primary" | "stress"; +export type PerformanceTestTaskKind = "single-tool" | "ordered-chain" | "near-match" | "no-tool"; +export type PerformanceTestTaskSuite = "primary" | "stress"; export interface ExpectedToolCall { tool_name: string; @@ -29,10 +30,10 @@ export interface ExpectedToolCall { result_nonce: string; } -export interface SyntheticBenchmarkTask { +export interface SyntheticPerformanceTask { id: string; - suite: BenchmarkTaskSuite; - kind: BenchmarkTaskKind; + suite: PerformanceTestTaskSuite; + kind: PerformanceTestTaskKind; prompt: string; min_catalog_size: number; expected_calls: ExpectedToolCall[]; @@ -41,12 +42,12 @@ export interface SyntheticBenchmarkTask { } export interface SyntheticTaskSet { - schema_version: typeof TOOL_DISCLOSURE_TASK_SCHEMA_VERSION; + schema_version: typeof TOOL_DISCLOSURE_PERFORMANCE_TEST_TASK_SCHEMA_VERSION; catalog_seed: string; - suite: BenchmarkTaskSuite; + suite: PerformanceTestTaskSuite; task_count: number; tasks_sha256: string; - tasks: SyntheticBenchmarkTask[]; + tasks: SyntheticPerformanceTask[]; } function toolAt(catalog: SyntheticCatalog, index: number): SyntheticTool { @@ -81,11 +82,11 @@ function positiveFinalInstruction(callCount: number): string { } function buildSingleTask( - suite: BenchmarkTaskSuite, + suite: PerformanceTestTaskSuite, ordinal: number, tool: SyntheticTool, variant: number, -): SyntheticBenchmarkTask { +): SyntheticPerformanceTask { const args = buildSyntheticArguments(tool, variant); const call = expectedCall(tool, args); return { @@ -108,7 +109,7 @@ function buildChainTask( ordinal: number, firstTool: SyntheticTool, secondTool: SyntheticTool, -): SyntheticBenchmarkTask { +): SyntheticPerformanceTask { const firstArgs = buildSyntheticArguments(firstTool, 200 + ordinal); const firstCall = expectedCall(firstTool, firstArgs); const secondArgs = buildSyntheticArguments(secondTool, 300 + ordinal, { @@ -140,7 +141,7 @@ function buildNearMatchTask( ordinal: number, target: SyntheticTool, distractors: readonly SyntheticTool[], -): SyntheticBenchmarkTask { +): SyntheticPerformanceTask { const args = buildSyntheticArguments(target, 400 + ordinal); const call = expectedCall(target, args); const distractorOperations = distractors @@ -183,7 +184,7 @@ const NO_TOOL_CONTROLS = [ }, ] as const; -function buildNoToolTasks(): SyntheticBenchmarkTask[] { +function buildNoToolTasks(): SyntheticPerformanceTask[] { return NO_TOOL_CONTROLS.map((control, index) => ({ id: `primary-no-tool-${String(index + 1).padStart(2, "0")}`, suite: "primary", @@ -196,7 +197,7 @@ function buildNoToolTasks(): SyntheticBenchmarkTask[] { })); } -export function buildPrimaryTasks(catalog: SyntheticCatalog): SyntheticBenchmarkTask[] { +export function buildPrimaryTasks(catalog: SyntheticCatalog): SyntheticPerformanceTask[] { if (catalog.size < PRIMARY_CATALOG_MIN_SIZE) { throw new RangeError( `primary tasks require at least ${PRIMARY_CATALOG_MIN_SIZE} tools; catalog has ${catalog.size}`, @@ -220,7 +221,7 @@ export function buildPrimaryTasks(catalog: SyntheticCatalog): SyntheticBenchmark const STRESS_TOOL_INDICES = [256, 511, 767, 1_023, 1_279, 1_535, 1_791, 2_208] as const; -export function buildStressTasks(catalog: SyntheticCatalog): SyntheticBenchmarkTask[] { +export function buildStressTasks(catalog: SyntheticCatalog): SyntheticPerformanceTask[] { if (catalog.size < STRESS_CATALOG_SIZE) { throw new RangeError( `stress tasks require ${STRESS_CATALOG_SIZE} tools; catalog has ${catalog.size}`, @@ -233,11 +234,11 @@ export function buildStressTasks(catalog: SyntheticCatalog): SyntheticBenchmarkT function buildTaskSet( catalog: SyntheticCatalog, - suite: BenchmarkTaskSuite, - tasks: SyntheticBenchmarkTask[], + suite: PerformanceTestTaskSuite, + tasks: SyntheticPerformanceTask[], ): SyntheticTaskSet { return { - schema_version: TOOL_DISCLOSURE_TASK_SCHEMA_VERSION, + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_TASK_SCHEMA_VERSION, catalog_seed: catalog.seed, suite, task_count: tasks.length, @@ -255,8 +256,8 @@ export function generateStressTaskSet(catalog: SyntheticCatalog): SyntheticTaskS } function expectedKindCounts( - suite: BenchmarkTaskSuite, -): Readonly> { + suite: PerformanceTestTaskSuite, +): Readonly> { return suite === "primary" ? { "single-tool": 8, "ordered-chain": 8, "near-match": 4, "no-tool": 4 } : { "single-tool": 8, "ordered-chain": 0, "near-match": 0, "no-tool": 0 }; @@ -267,7 +268,7 @@ export function validateSyntheticTaskSet( catalog: SyntheticCatalog, ): string[] { const errors: string[] = []; - if (taskSet.schema_version !== TOOL_DISCLOSURE_TASK_SCHEMA_VERSION) { + if (taskSet.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_TASK_SCHEMA_VERSION) { errors.push(`unsupported task schema version: ${taskSet.schema_version}`); } if (taskSet.catalog_seed !== catalog.seed) errors.push("task and catalog seeds do not match"); @@ -277,7 +278,7 @@ export function validateSyntheticTaskSet( const ids = new Set(); const tools = new Map(catalog.tools.map((tool) => [tool.definition.function.name, tool])); - const counts: Record = { + const counts: Record = { "single-tool": 0, "ordered-chain": 0, "near-match": 0, @@ -343,7 +344,7 @@ export function validateSyntheticTaskSet( } const expectedCounts = expectedKindCounts(taskSet.suite); - for (const kind of Object.keys(counts) as BenchmarkTaskKind[]) { + for (const kind of Object.keys(counts) as PerformanceTestTaskKind[]) { if (counts[kind] !== expectedCounts[kind]) { errors.push( `${taskSet.suite} task set has ${counts[kind]} ${kind} tasks; expected ${expectedCounts[kind]}`, diff --git a/scripts/bench/tool-disclosure/telemetry.ts b/scripts/performance/tool-disclosure/telemetry.ts similarity index 100% rename from scripts/bench/tool-disclosure/telemetry.ts rename to scripts/performance/tool-disclosure/telemetry.ts diff --git a/scripts/bench/tool-disclosure/types.ts b/scripts/performance/tool-disclosure/types.ts similarity index 84% rename from scripts/bench/tool-disclosure/types.ts rename to scripts/performance/tool-disclosure/types.ts index 60a11a3578..4dca7fa51a 100644 --- a/scripts/bench/tool-disclosure/types.ts +++ b/scripts/performance/tool-disclosure/types.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Public evidence schema for the progressive tool-disclosure benchmark. + * Public evidence schema for the progressive tool-disclosure performance test. * * The schema deliberately stores task identifiers, classifications, booleans, * and numeric measurements rather than prompts, tool arguments, response @@ -10,19 +10,25 @@ * checksummed artifacts and never flow through the statistics/report layer. */ -export const TOOL_DISCLOSURE_SCHEMA_VERSION = "nemoclaw.tool_disclosure_bench.v1" as const; +export const TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION = + "nemoclaw.tool_disclosure_performance_test.v1" as const; export const DEFAULT_BOOTSTRAP_SAMPLES = 10_000; export const DEFAULT_BOOTSTRAP_SEED = 0x4e_56_44_41; export const DEFAULT_NONINFERIORITY_MARGIN_PP = -5; export const NO_ACCELERATOR_TYPE = "none" as const; export const NOT_APPLICABLE = "not-applicable" as const; -export type ToolDisclosureSchemaVersion = typeof TOOL_DISCLOSURE_SCHEMA_VERSION; +export type ToolDisclosurePerformanceTestSchemaVersion = + typeof TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION; export type ToolDisclosureAgent = "langchain-deepagents-code" | "hermes" | "openclaw"; export type ToolDisclosureMode = "direct" | "progressive"; -export type BenchmarkPhase = "static-visibility" | "small-control" | "primary" | "large-stress"; -export type BenchmarkTaskKind = "single-tool" | "ordered-chain" | "near-match" | "no-tool"; -export type BenchmarkRunOutcome = +export type PerformanceTestPhase = + | "static-visibility" + | "small-control" + | "primary" + | "large-stress"; +export type PerformanceTestTaskKind = "single-tool" | "ordered-chain" | "near-match" | "no-tool"; +export type PerformanceTestRunOutcome = | "success" | "incorrect" | "timeout" @@ -44,7 +50,7 @@ export interface CampaignDefinition { } /** Public, sanitized machine metadata. There is intentionally no hostname or path field. */ -export interface BenchmarkEnvironment { +export interface PerformanceTestEnvironment { operating_system: string; architecture: string; cpu_model: string; @@ -80,16 +86,16 @@ export interface InferenceConfiguration { public_vllm_flags: readonly string[]; } -export interface BenchmarkProtocol { +export interface PerformanceTestProtocol { agents: readonly ToolDisclosureAgent[]; modes: readonly ToolDisclosureMode[]; catalog_sizes: readonly number[]; primary_catalog_size: number; tasks: readonly { task_id: string; - kind: BenchmarkTaskKind; + kind: PerformanceTestTaskKind; }[]; - repetitions: Readonly, number>>; + repetitions: Readonly, number>>; execution_seed: number; bootstrap_samples: number; bootstrap_seed: number; @@ -98,14 +104,14 @@ export interface BenchmarkProtocol { } export interface ToolDisclosureManifest { - schema_version: ToolDisclosureSchemaVersion; - benchmark_id: string; + schema_version: ToolDisclosurePerformanceTestSchemaVersion; + performance_test_id: string; created_at: string; sut: RevisionIdentity; harness: RevisionIdentity; campaigns: readonly CampaignDefinition[]; - protocol: BenchmarkProtocol; - environment: BenchmarkEnvironment; + protocol: PerformanceTestProtocol; + environment: PerformanceTestEnvironment; inference: InferenceConfiguration; } @@ -137,19 +143,19 @@ export interface RunMeasurements { } export interface ToolDisclosureRun { - schema_version: ToolDisclosureSchemaVersion; - benchmark_id: string; + schema_version: ToolDisclosurePerformanceTestSchemaVersion; + performance_test_id: string; campaign_id: string; run_id: string; - phase: BenchmarkPhase; + phase: PerformanceTestPhase; agent: ToolDisclosureAgent; mode: ToolDisclosureMode; catalog_size: number; task_id: string; - task_kind?: BenchmarkTaskKind; + task_kind?: PerformanceTestTaskKind; repetition: number; execution_seed: number; - outcome: BenchmarkRunOutcome; + outcome: PerformanceTestRunOutcome; /** Static captures are unscored; for task phases, only setup-error is excluded. */ scored: boolean; correctness: RunCorrectness; @@ -188,7 +194,7 @@ export interface ComparisonDifferences { export interface ComparisonCellSummary { campaign_id: string; - phase: Exclude; + phase: Exclude; agent: ToolDisclosureAgent; catalog_size: number; direct: ModeAggregate; @@ -238,8 +244,8 @@ export interface ClaimGateSummary { } export interface ToolDisclosureSummary { - schema_version: ToolDisclosureSchemaVersion; - benchmark_id: string; + schema_version: ToolDisclosurePerformanceTestSchemaVersion; + performance_test_id: string; generated_at: string; bootstrap_samples: number; bootstrap_seed: number; @@ -278,8 +284,8 @@ export interface EvidenceArtifact { } export interface EvidenceBundle { - schema_version: ToolDisclosureSchemaVersion; - benchmark_id: string; + schema_version: ToolDisclosurePerformanceTestSchemaVersion; + performance_test_id: string; generated_at: string; artifacts: readonly EvidenceArtifact[]; } diff --git a/scripts/bench/tool-disclosure/validation.ts b/scripts/performance/tool-disclosure/validation.ts similarity index 98% rename from scripts/bench/tool-disclosure/validation.ts rename to scripts/performance/tool-disclosure/validation.ts index 3d58d9bcf4..e4a769d6dd 100644 --- a/scripts/bench/tool-disclosure/validation.ts +++ b/scripts/performance/tool-disclosure/validation.ts @@ -117,8 +117,8 @@ export function validateFrozenManifest(manifest: ToolDisclosureManifest): void { manifest.inference.reasoning_parser, ]; if ( - manifest.schema_version !== "nemoclaw.tool_disclosure_bench.v1" || - !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/u.test(manifest.benchmark_id) || + manifest.schema_version !== "nemoclaw.tool_disclosure_performance_test.v1" || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/u.test(manifest.performance_test_id) || !Number.isFinite(Date.parse(manifest.created_at)) || !/^[a-f0-9]{40}$/u.test(manifest.sut.git_sha) || !/^[a-f0-9]{40}$/u.test(manifest.harness.git_sha) || @@ -176,7 +176,7 @@ export function validateFrozenManifest(manifest: ToolDisclosureManifest): void { DEFAULT_NONINFERIORITY_MARGIN_PP || manifest.protocol.retry_setup_failures !== 1 ) { - throw new Error("manifest does not match the frozen benchmark protocol"); + throw new Error("manifest does not match the frozen performance test protocol"); } if ( manifest.inference.temperature !== 0 || @@ -319,7 +319,7 @@ export function validateCompleteEvidence(options: { throw new Error("manifest inference controls do not match the frozen protocol"); } if (!manifest.sut.worktree_clean || !manifest.harness.worktree_clean) { - throw new Error("public benchmark evidence requires a clean SUT and harness worktree"); + throw new Error("public performance test evidence requires a clean SUT and harness worktree"); } const imageKeys = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => TOOL_DISCLOSURE_MODES.flatMap((mode) => diff --git a/test/e2e/live/tool-disclosure-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts similarity index 76% rename from test/e2e/live/tool-disclosure-smoke.test.ts rename to test/e2e/live/tool-disclosure-performance-smoke.test.ts index 255afc069c..76e4e53388 100644 --- a/test/e2e/live/tool-disclosure-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -5,19 +5,19 @@ import { execFileSync } from "node:child_process"; import { randomBytes } from "node:crypto"; import { - DEFAULT_SYNTHETIC_CATALOG_SEED, + DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, generateCatalogPrefix, generateSyntheticCatalog, -} from "../../../scripts/bench/tool-disclosure/catalog"; +} from "../../../scripts/performance/tool-disclosure/catalog"; import { buildAgentDriverCommand, extractFinalAssistantOutput, -} from "../../../scripts/bench/tool-disclosure/drivers"; -import { gradeTaskRun } from "../../../scripts/bench/tool-disclosure/grading"; -import { SyntheticMcpServer } from "../../../scripts/bench/tool-disclosure/mcp-server"; -import { startQuickTunnel } from "../../../scripts/bench/tool-disclosure/quick-tunnel"; -import type { ToolDisclosureMode } from "../../../scripts/bench/tool-disclosure/schedule"; -import { generatePrimaryTaskSet } from "../../../scripts/bench/tool-disclosure/tasks"; +} from "../../../scripts/performance/tool-disclosure/drivers"; +import { gradeTaskRun } from "../../../scripts/performance/tool-disclosure/grading"; +import { SyntheticMcpServer } from "../../../scripts/performance/tool-disclosure/mcp-server"; +import { startQuickTunnel } from "../../../scripts/performance/tool-disclosure/quick-tunnel"; +import type { ToolDisclosureMode } from "../../../scripts/performance/tool-disclosure/schedule"; +import { generatePrimaryTaskSet } from "../../../scripts/performance/tool-disclosure/tasks"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -28,12 +28,12 @@ const AGENT = "langchain-deepagents-code" as const; const CATALOG_SIZE = 64; const TASK_ID = "primary-single-01"; const MODES = ["progressive", "direct"] as const satisfies readonly ToolDisclosureMode[]; -const MCP_SERVER_NAME = "benchmark"; -const MCP_TOKEN_ENV = "NEMOCLAW_BENCHMARK_MCP_BEARER"; +const MCP_SERVER_NAME = "performance-test"; +const MCP_TOKEN_ENV = "NEMOCLAW_PERFORMANCE_TEST_MCP_BEARER"; const TEST_TIMEOUT_MS = 50 * 60_000; function sandboxName(mode: ToolDisclosureMode): string { - return `e2e-tool-disclosure-${mode}`; + return `e2e-tool-disclosure-performance-${mode}`; } function hostEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { @@ -50,30 +50,35 @@ function gitSha(): string { execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8", }).trim(); - if (!/^[a-f0-9]{40}$/u.test(sha)) throw new Error("tool-disclosure smoke requires a git SHA"); + if (!/^[a-f0-9]{40}$/u.test(sha)) { + throw new Error("tool-disclosure performance smoke requires a git SHA"); + } return sha; } test.skipIf(!shouldRunLiveE2E())( - "tool disclosure hosted-inference smoke completes one frozen task in both modes", + "tool disclosure hosted-inference performance smoke completes one frozen task in both modes", { timeout: TEST_TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox, secrets }) => { const hosted = requireHostedInferenceConfig(secrets); const bearerToken = randomBytes(32).toString("hex"); artifacts.addRedactionValues([hosted.apiKey, bearerToken]); - const catalog = generateSyntheticCatalog({ seed: DEFAULT_SYNTHETIC_CATALOG_SEED }); + const catalog = generateSyntheticCatalog({ + seed: DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, + }); const catalogPrefix = generateCatalogPrefix(catalog, CATALOG_SIZE); const primaryTasks = generatePrimaryTaskSet(catalog); const task = primaryTasks.tasks.find((candidate) => candidate.id === TASK_ID); if (!task) throw new Error(`frozen task ${TASK_ID} is missing`); if (task.min_catalog_size > catalogPrefix.size) { - throw new Error(`frozen task ${TASK_ID} does not fit the smoke catalog`); + throw new Error(`frozen task ${TASK_ID} does not fit the performance smoke catalog`); } await artifacts.writeJson("scenario.json", { - id: "tool-disclosure-smoke", - boundary: "two real Deep Agents Code sandboxes with hosted inference and synthetic MCP", + id: "tool-disclosure-performance-smoke", + boundary: + "tool-disclosure performance smoke with two real Deep Agents Code sandboxes, hosted inference, and synthetic MCP", claim_eligible: false, agent: AGENT, modes: MODES, @@ -84,13 +89,13 @@ test.skipIf(!shouldRunLiveE2E())( const mcp = new SyntheticMcpServer(catalogPrefix, bearerToken); const mcpAddress = await mcp.start(); - cleanup.add("stop tool-disclosure synthetic MCP server", () => mcp.stop()); + cleanup.add("stop tool-disclosure performance synthetic MCP server", () => mcp.stop()); const tunnel = await startQuickTunnel({ port: mcpAddress.port }); - cleanup.add("stop tool-disclosure MCP quick tunnel", () => tunnel.close()); + cleanup.add("stop tool-disclosure performance MCP quick tunnel", () => tunnel.close()); for (const mode of MODES) { const name = sandboxName(mode); - cleanup.add(`destroy ${mode} tool-disclosure smoke sandbox`, () => + cleanup.add(`destroy ${mode} tool-disclosure performance smoke sandbox`, () => host.bestEffortCleanupSandbox(name, { artifactName: `cleanup-${mode}-tool-disclosure-sandbox`, env: hostEnv(), @@ -166,7 +171,7 @@ test.skipIf(!shouldRunLiveE2E())( const results = []; for (const mode of MODES) { - const runId = `smoke-${AGENT}-${mode}-${TASK_ID}`; + const runId = `performance-smoke-${AGENT}-${mode}-${TASK_ID}`; const driver = buildAgentDriverCommand({ openshellBin: process.env.OPENSHELL_BIN, sandboxName: sandboxName(mode), @@ -210,8 +215,8 @@ test.skipIf(!shouldRunLiveE2E())( }); } - await artifacts.writeJson("tool-disclosure-smoke.json", { - schema_version: "nemoclaw.tool_disclosure_smoke.v1", + await artifacts.writeJson("tool-disclosure-performance-smoke.json", { + schema_version: "nemoclaw.tool_disclosure_performance_smoke.v1", generated_at: new Date().toISOString(), claim_eligible: false, sut_git_sha: gitSha(), @@ -233,16 +238,16 @@ test.skipIf(!shouldRunLiveE2E())( }, results, limitations: [ - "This smoke verifies live wiring and task completion; it is not the full benchmark.", + "This performance smoke test verifies live wiring and task completion; it is not the complete two-campaign performance test.", "One observation per mode is insufficient for performance or quality claims.", "The fixed progressive-then-direct order makes elapsed times informational and subject to cold-cache and ordering effects.", - "The smoke does not collect vLLM tokenizer, token-counter, or request-recorder evidence.", + "The performance smoke test does not collect vLLM tokenizer, token-counter, or request-recorder evidence.", ], }); expect(results).toHaveLength(2); for (const result of results) { - expect(result.outcome, `${result.mode} smoke outcome`).toBe("success"); + expect(result.outcome, `${result.mode} performance smoke outcome`).toBe("success"); expect(result.correctness.task_success, `${result.mode} task correctness`).toBe(true); expect(result.synthetic_call_count, `${result.mode} synthetic call count`).toBe(1); } diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 8dd50171a2..a5a1963fab 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -23,7 +23,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { expect(inventory.allowedJobs).toContain("jetson-nvmap-gpu"); expect(inventory.explicitOnlyJobs).toContain("jetson-nvmap-gpu"); expect(formatFreeStandingJobsInventoryForShell(inventory)).toContain( - "explicit_only_jobs_csv=openshell-gateway-auth-contract,mcp-bridge-dev,tool-disclosure-smoke,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", + "explicit_only_jobs_csv=openshell-gateway-auth-contract,mcp-bridge-dev,tool-disclosure-performance-smoke,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", ); expect(inventory.targetToJob.get("jetson-nvmap-gpu")).toBe("jetson-nvmap-gpu"); expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( diff --git a/test/e2e/support/tool-disclosure-smoke-workflow-boundary.test.ts b/test/e2e/support/tool-disclosure-performance-smoke-workflow-boundary.test.ts similarity index 72% rename from test/e2e/support/tool-disclosure-smoke-workflow-boundary.test.ts rename to test/e2e/support/tool-disclosure-performance-smoke-workflow-boundary.test.ts index 23405e9cd0..5682d512db 100644 --- a/test/e2e/support/tool-disclosure-smoke-workflow-boundary.test.ts +++ b/test/e2e/support/tool-disclosure-performance-smoke-workflow-boundary.test.ts @@ -30,9 +30,9 @@ type WorkflowJob = { "timeout-minutes"?: number; }; -function smokeJob(): WorkflowJob { +function performanceSmokeJob(): WorkflowJob { const workflow = readWorkflow() as { jobs: Record }; - const job = workflow.jobs["tool-disclosure-smoke"]; + const job = workflow.jobs["tool-disclosure-performance-smoke"]; expect(job).toBeDefined(); return job; } @@ -41,44 +41,47 @@ function namedStep(job: WorkflowJob, name: string): WorkflowStep | undefined { return job.steps?.find((step) => step.name === name); } -describe("tool-disclosure smoke workflow boundary", () => { +describe("tool-disclosure performance smoke workflow boundary", () => { it("keeps the hosted-inference smoke explicit-only", () => { const inventory = readFreeStandingJobsInventory(); expect(validateE2eWorkflowBoundary()).toEqual([]); - expect(inventory.allowedJobs).toContain("tool-disclosure-smoke"); - expect(inventory.explicitOnlyJobs).toContain("tool-disclosure-smoke"); - expect(inventory.targetToJob.get("tool-disclosure-smoke")).toBe("tool-disclosure-smoke"); + expect(inventory.allowedJobs).toContain("tool-disclosure-performance-smoke"); + expect(inventory.explicitOnlyJobs).toContain("tool-disclosure-performance-smoke"); + expect(inventory.targetToJob.get("tool-disclosure-performance-smoke")).toBe( + "tool-disclosure-performance-smoke", + ); expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( - "tool-disclosure-smoke", + "tool-disclosure-performance-smoke", ); for (const selector of [ - { jobs: "tool-disclosure-smoke" }, - { targets: "tool-disclosure-smoke" }, + { jobs: "tool-disclosure-performance-smoke" }, + { targets: "tool-disclosure-performance-smoke" }, ]) { expect(evaluateE2eWorkflowDispatchSelectors(selector)).toMatchObject({ valid: true, liveTargetsRun: false, - selectedFreeStandingJobs: ["tool-disclosure-smoke"], + selectedFreeStandingJobs: ["tool-disclosure-performance-smoke"], registryTargets: [], }); } }); it("runs the focused live test with reviewed prerequisites and scoped secrets", () => { - const job = smokeJob(); + const job = performanceSmokeJob(); expect(job).toMatchObject({ needs: "generate-matrix", - if: "${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-smoke,') }}", + if: "${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-performance-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-performance-smoke,') }}", "runs-on": "ubuntu-latest", permissions: { contents: "read" }, "timeout-minutes": 60, env: { E2E_JOB: "1", E2E_DEFAULT_ENABLED: "0", - E2E_TARGET_ID: "tool-disclosure-smoke", - E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-smoke", + E2E_TARGET_ID: "tool-disclosure-performance-smoke", + E2E_ARTIFACT_DIR: + "${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-performance-smoke", NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", NEMOCLAW_RUN_LIVE_E2E: "1", NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", @@ -105,25 +108,25 @@ describe("tool-disclosure smoke workflow boundary", () => { expect(cloudflared?.run).toContain("sha256sum -c -"); expect(cloudflared?.run).toContain("dpkg-deb -f"); - const run = namedStep(job, "Run tool-disclosure smoke live test"); + const run = namedStep(job, "Run tool-disclosure performance smoke live test"); expect(run?.env).toEqual({ NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", }); - expect(run?.run).toContain("test/e2e/live/tool-disclosure-smoke.test.ts"); + expect(run?.run).toContain("test/e2e/live/tool-disclosure-performance-smoke.test.ts"); expect(run?.run).toContain("--project e2e-live"); }); it("maps only the smoke artifact directory through the reviewed uploader", () => { - const job = smokeJob(); - const upload = namedStep(job, "Upload tool-disclosure smoke artifacts"); + const job = performanceSmokeJob(); + const upload = namedStep(job, "Upload tool-disclosure performance smoke artifacts"); expect(upload).toEqual({ - name: "Upload tool-disclosure smoke artifacts", + name: "Upload tool-disclosure performance smoke artifacts", if: "always()", uses: UPLOAD_E2E_ARTIFACTS_ACTION, }); - expect(job.env?.E2E_TARGET_ID).toBe("tool-disclosure-smoke"); + expect(job.env?.E2E_TARGET_ID).toBe("tool-disclosure-performance-smoke"); expect(job.env?.E2E_ARTIFACT_DIR).toBe( - "${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-smoke", + "${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-performance-smoke", ); expect(job.steps?.at(-1)?.name).toBe("Clean up Docker auth"); }); diff --git a/test/bench/tool-disclosure-catalog.test.ts b/test/performance/tool-disclosure-catalog.test.ts similarity index 97% rename from test/bench/tool-disclosure-catalog.test.ts rename to test/performance/tool-disclosure-catalog.test.ts index 95ea33473c..0a23116e3b 100644 --- a/test/bench/tool-disclosure-catalog.test.ts +++ b/test/performance/tool-disclosure-catalog.test.ts @@ -7,7 +7,7 @@ import { assertValidSyntheticCatalog, buildSyntheticArguments, canonicalJson, - DEFAULT_SYNTHETIC_CATALOG_SEED, + DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, executeSyntheticTool, generateCatalogPrefix, generateCatalogPrefixes, @@ -20,7 +20,7 @@ import { toOpenAIFunctionTools, validateSyntheticArguments, validateSyntheticCatalog, -} from "../../scripts/bench/tool-disclosure/catalog"; +} from "../../scripts/performance/tool-disclosure/catalog"; import { assertValidSyntheticTaskSet, buildPrimaryTasks, @@ -32,7 +32,7 @@ import { STRESS_CATALOG_SIZE, STRESS_TASK_COUNT, validateSyntheticTaskSet, -} from "../../scripts/bench/tool-disclosure/tasks"; +} from "../../scripts/performance/tool-disclosure/tasks"; let fullCatalog: SyntheticCatalog; @@ -54,7 +54,7 @@ describe("synthetic tool-disclosure catalog", () => { }); it("generates 2,209 unique, safe OpenAI-style function definitions", () => { - expect(fullCatalog.seed).toBe(DEFAULT_SYNTHETIC_CATALOG_SEED); + expect(fullCatalog.seed).toBe(DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED); expect(fullCatalog.size).toBe(MAX_SYNTHETIC_TOOLS); expect(fullCatalog.tools).toHaveLength(2_209); expect(validateSyntheticCatalog(fullCatalog)).toEqual([]); diff --git a/test/bench/tool-disclosure-drivers.test.ts b/test/performance/tool-disclosure-drivers.test.ts similarity index 76% rename from test/bench/tool-disclosure-drivers.test.ts rename to test/performance/tool-disclosure-drivers.test.ts index 8618a4925c..87625259b0 100644 --- a/test/bench/tool-disclosure-drivers.test.ts +++ b/test/performance/tool-disclosure-drivers.test.ts @@ -9,7 +9,7 @@ import { buildAgentDriverCommand, extractFinalAssistantOutput, outputContainsOracle, -} from "../../scripts/bench/tool-disclosure/drivers"; +} from "../../scripts/performance/tool-disclosure/drivers"; describe("tool-disclosure agent drivers", () => { it.each([ @@ -17,19 +17,19 @@ describe("tool-disclosure agent drivers", () => { "hermes", "langchain-deepagents-code", ] as const)("builds a bounded sandbox command for %s", (agent) => { - const prompt = "Use the weather converter and return BENCH_OK_123"; + const prompt = "Use the weather converter and return PERFORMANCE_TEST_OK_123"; const command = buildAgentDriverCommand({ agent, - sandboxName: "bench-openclaw-progressive-512", + sandboxName: "performance-test-openclaw-progressive-512", prompt, - sessionId: "bench-session-1", + sessionId: "performance-test-session-1", }); expect(command.command).toBe("openshell"); expect(command.args.slice(0, 5)).toEqual([ "sandbox", "exec", "-n", - "bench-openclaw-progressive-512", + "performance-test-openclaw-progressive-512", "--", ]); expect(command.args.join(" ")).not.toContain(prompt); @@ -37,8 +37,12 @@ describe("tool-disclosure agent drivers", () => { }); it("uses an exact oracle and rejects unsafe identifiers", () => { - expect(outputContainsOracle("result BENCH_OK_123", "BENCH_OK_123")).toBe(true); - expect(outputContainsOracle("result BENCH_OK_123", "BENCH_OK_456")).toBe(false); + expect(outputContainsOracle("result PERFORMANCE_TEST_OK_123", "PERFORMANCE_TEST_OK_123")).toBe( + true, + ); + expect(outputContainsOracle("result PERFORMANCE_TEST_OK_123", "PERFORMANCE_TEST_OK_456")).toBe( + false, + ); expect(() => buildAgentDriverCommand({ agent: "openclaw", @@ -53,9 +57,9 @@ describe("tool-disclosure agent drivers", () => { for (const agent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { const command = buildAgentDriverCommand({ agent, - sandboxName: "bench-syntax", - prompt: "Return BENCH_OK_123", - sessionId: "bench-session-syntax", + sandboxName: "performance-test-syntax", + prompt: "Return PERFORMANCE_TEST_OK_123", + sessionId: "performance-test-session-syntax", }); const syntax = spawnSync("sh", ["-n"], { encoding: "utf8", @@ -68,9 +72,9 @@ describe("tool-disclosure agent drivers", () => { it("streams the decoded Hermes payload directly to curl", () => { const command = buildAgentDriverCommand({ agent: "hermes", - sandboxName: "bench-hermes-stream", - prompt: "Return BENCH_OK_123", - sessionId: "bench-session-hermes-stream", + sandboxName: "performance-test-hermes-stream", + prompt: "Return PERFORMANCE_TEST_OK_123", + sessionId: "performance-test-session-hermes-stream", }); const script = command.args.at(-1) ?? ""; expect(script).toContain("base64 -d | curl"); @@ -107,7 +111,7 @@ describe("tool-disclosure agent drivers", () => { "✓ Loaded 1 MCP tool", "✓ Server ready", "🔧 Calling tool: search_tools (trace: TOOL_ONLY_NONCE)", - "🔧 Calling tool: benchmark_echo", + "🔧 Calling tool: performance_test_echo", "Operation complete.", "FINAL_NONCE", "", @@ -122,7 +126,7 @@ describe("tool-disclosure agent drivers", () => { expect( extractFinalAssistantOutput( "langchain-deepagents-code", - "✓ Server ready\n🔧 Calling tool: benchmark_echo (result: TOOL_ONLY_NONCE)\n✓ Task completed\nAgent active 1.3s", + "✓ Server ready\n🔧 Calling tool: performance_test_echo (result: TOOL_ONLY_NONCE)\n✓ Task completed\nAgent active 1.3s", ), ).toBe(""); }); diff --git a/test/bench/tool-disclosure-execute.test.ts b/test/performance/tool-disclosure-execute.test.ts similarity index 96% rename from test/bench/tool-disclosure-execute.test.ts rename to test/performance/tool-disclosure-execute.test.ts index 6040a361ad..04d55b0bca 100644 --- a/test/bench/tool-disclosure-execute.test.ts +++ b/test/performance/tool-disclosure-execute.test.ts @@ -17,14 +17,14 @@ import { readAttestedVllmConfiguration, recoverAttemptJournal, selectInspectedContainer, -} from "../../scripts/bench/tool-disclosure/execute"; -import { validateCampaignAttestations } from "../../scripts/bench/tool-disclosure/run"; +} from "../../scripts/performance/tool-disclosure/execute"; +import { validateCampaignAttestations } from "../../scripts/performance/tool-disclosure/run"; import { STATIC_CATALOG_SIZES, TOOL_DISCLOSURE_AGENTS, TOOL_DISCLOSURE_MODES, -} from "../../scripts/bench/tool-disclosure/schedule"; -import type { ToolDisclosureManifest } from "../../scripts/bench/tool-disclosure/types"; +} from "../../scripts/performance/tool-disclosure/schedule"; +import type { ToolDisclosureManifest } from "../../scripts/performance/tool-disclosure/types"; const hex = (value: number): string => value.toString(16).padStart(64, "0"); diff --git a/test/bench/tool-disclosure-fixture.test.ts b/test/performance/tool-disclosure-fixture.test.ts similarity index 90% rename from test/bench/tool-disclosure-fixture.test.ts rename to test/performance/tool-disclosure-fixture.test.ts index b0769d1f75..5598e71e7c 100644 --- a/test/bench/tool-disclosure-fixture.test.ts +++ b/test/performance/tool-disclosure-fixture.test.ts @@ -8,37 +8,37 @@ import { pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; -import { assembleToolDisclosureRun } from "../../scripts/bench/tool-disclosure/assemble-run"; +import { assembleToolDisclosureRun } from "../../scripts/performance/tool-disclosure/assemble-run"; import { canonicalJson, executeSyntheticTool, generateSyntheticCatalog, sha256Hex, -} from "../../scripts/bench/tool-disclosure/catalog"; -import { gradeTaskRun } from "../../scripts/bench/tool-disclosure/grading"; +} from "../../scripts/performance/tool-disclosure/catalog"; +import { gradeTaskRun } from "../../scripts/performance/tool-disclosure/grading"; import { assertDockerfileSandboxBase, writeOpenClawFixture, -} from "../../scripts/bench/tool-disclosure/openclaw-fixture"; -import { buildToolDisclosureSchedule } from "../../scripts/bench/tool-disclosure/schedule"; -import { buildPrimaryTasks } from "../../scripts/bench/tool-disclosure/tasks"; +} from "../../scripts/performance/tool-disclosure/openclaw-fixture"; +import { buildToolDisclosureSchedule } from "../../scripts/performance/tool-disclosure/schedule"; +import { buildPrimaryTasks } from "../../scripts/performance/tool-disclosure/tasks"; import { - TOOL_DISCLOSURE_SCHEMA_VERSION, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, type ToolDisclosureManifest, -} from "../../scripts/bench/tool-disclosure/types"; +} from "../../scripts/performance/tool-disclosure/types"; const temporaryDirectories: string[] = []; afterEach(() => { - delete process.env.NEMOCLAW_BENCH_CALLS_PATH; + delete process.env.NEMOCLAW_PERFORMANCE_TEST_CALLS_PATH; for (const directory of temporaryDirectories.splice(0)) { fs.rmSync(directory, { recursive: true, force: true }); } }); -describe("OpenClaw benchmark fixture", () => { +describe("OpenClaw performance test fixture", () => { it("registers and executes the deterministic native tool catalog", async () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bench-fixture-")); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-performance-test-fixture-")); temporaryDirectories.push(root); const output = path.join(root, "fixture-16"); const catalog = generateSyntheticCatalog({ size: 16 }); @@ -50,7 +50,7 @@ describe("OpenClaw benchmark fixture", () => { const registered: Array> = []; const callsPath = path.join(root, "calls.jsonl"); - process.env.NEMOCLAW_BENCH_CALLS_PATH = callsPath; + process.env.NEMOCLAW_PERFORMANCE_TEST_CALLS_PATH = callsPath; const moduleUrl = `${pathToFileURL(path.join(output, "plugin", "index.js")).href}?test=1`; const plugin = (await import(moduleUrl)) as { default(api: { registerTool(tool: Record): void }): void; @@ -82,7 +82,7 @@ describe("OpenClaw benchmark fixture", () => { }); it("rejects Dockerfile injection before creating the fixture", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bench-fixture-")); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-performance-test-fixture-")); temporaryDirectories.push(root); const output = path.join(root, "fixture-injected"); const injected = `registry.example/base\nRUN touch /tmp/injected\n#@sha256:${"1".repeat(64)}`; @@ -115,7 +115,7 @@ describe("OpenClaw benchmark fixture", () => { }); }); -describe("benchmark grading", () => { +describe("performance test grading", () => { it("requires exact ordered calls, argument hashes, nonces, and final oracle", () => { const catalog = generateSyntheticCatalog({ size: 64 }); const task = buildPrimaryTasks(catalog).find((candidate) => candidate.kind === "ordered-chain"); @@ -180,8 +180,8 @@ describe("benchmark grading", () => { expect(scheduled).toBeDefined(); const requiredScheduled = scheduled as NonNullable; const manifest = { - schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: "fixture-benchmark", + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: "fixture-performance-test", campaigns: [ { campaign_id: "campaign-1", diff --git a/test/bench/tool-disclosure-mcp-server.test.ts b/test/performance/tool-disclosure-mcp-server.test.ts similarity index 93% rename from test/bench/tool-disclosure-mcp-server.test.ts rename to test/performance/tool-disclosure-mcp-server.test.ts index 58e1422365..eeac1adbdf 100644 --- a/test/bench/tool-disclosure-mcp-server.test.ts +++ b/test/performance/tool-disclosure-mcp-server.test.ts @@ -6,11 +6,11 @@ import { describe, expect, it } from "vitest"; import { buildSyntheticArguments, generateSyntheticCatalog, -} from "../../scripts/bench/tool-disclosure/catalog"; +} from "../../scripts/performance/tool-disclosure/catalog"; import { executeArgumentHash, SyntheticMcpServer, -} from "../../scripts/bench/tool-disclosure/mcp-server"; +} from "../../scripts/performance/tool-disclosure/mcp-server"; async function rpc( url: string, @@ -25,7 +25,7 @@ async function rpc( }); } -describe("synthetic benchmark MCP server", () => { +describe("synthetic performance test MCP server", () => { it("lists and executes the generated catalog while retaining content-free call evidence", async () => { const catalog = generateSyntheticCatalog({ size: 16, seed: "mcp-server-test" }); const token = "fixture-token-that-must-not-be-recorded"; @@ -39,7 +39,7 @@ describe("synthetic benchmark MCP server", () => { protocolVersion: "2025-03-26", }); expect((await initialized.json()) as object).toMatchObject({ - result: { serverInfo: { name: "nemoclaw-tool-disclosure-benchmark" } }, + result: { serverInfo: { name: "nemoclaw-tool-disclosure-performance-test" } }, }); const listed = await rpc(address.local_url, token, "tools/list"); diff --git a/test/bench/tool-disclosure-quick-tunnel.test.ts b/test/performance/tool-disclosure-quick-tunnel.test.ts similarity index 86% rename from test/bench/tool-disclosure-quick-tunnel.test.ts rename to test/performance/tool-disclosure-quick-tunnel.test.ts index 7634f84e11..0cb00660cd 100644 --- a/test/bench/tool-disclosure-quick-tunnel.test.ts +++ b/test/performance/tool-disclosure-quick-tunnel.test.ts @@ -7,13 +7,15 @@ import { buildQuickTunnelArgs, buildQuickTunnelEnvironment, parseQuickTunnelOrigin, -} from "../../scripts/bench/tool-disclosure/quick-tunnel"; +} from "../../scripts/performance/tool-disclosure/quick-tunnel"; describe("tool-disclosure quick tunnel", () => { it("extracts only a bounded trycloudflare origin", () => { expect( - parseQuickTunnelOrigin("INF Requesting new quick Tunnel https://bench-123.trycloudflare.com"), - ).toBe("https://bench-123.trycloudflare.com"); + parseQuickTunnelOrigin( + "INF Requesting new quick Tunnel https://performance-test-123.trycloudflare.com", + ), + ).toBe("https://performance-test-123.trycloudflare.com"); expect(parseQuickTunnelOrigin("https://example.com/secret")).toBeNull(); }); diff --git a/test/bench/tool-disclosure-recorder.test.ts b/test/performance/tool-disclosure-recorder.test.ts similarity index 99% rename from test/bench/tool-disclosure-recorder.test.ts rename to test/performance/tool-disclosure-recorder.test.ts index 93a7977376..8f44fe6af9 100644 --- a/test/bench/tool-disclosure-recorder.test.ts +++ b/test/performance/tool-disclosure-recorder.test.ts @@ -10,7 +10,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { createToolDisclosureRecordingProxy, type ToolDisclosureRecordingProxy, -} from "../../scripts/bench/tool-disclosure/recorder"; +} from "../../scripts/performance/tool-disclosure/recorder"; interface TestUpstream { baseUrl: string; @@ -99,7 +99,7 @@ describe("tool-disclosure recording proxy", () => { ).toThrow("upstreamBaseUrl is allowed only on loopback"); }); - it("rejects non-loopback HTTPS recorder upstreams for the local benchmark", () => { + it("rejects non-loopback HTTPS recorder upstreams for the local performance test", () => { expect(() => createToolDisclosureRecordingProxy({ upstreamBaseUrl: "https://inference.example/v1", diff --git a/test/bench/tool-disclosure-report.test.ts b/test/performance/tool-disclosure-report.test.ts similarity index 95% rename from test/bench/tool-disclosure-report.test.ts rename to test/performance/tool-disclosure-report.test.ts index 0d6c15363f..e95a5d2565 100644 --- a/test/bench/tool-disclosure-report.test.ts +++ b/test/performance/tool-disclosure-report.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; -import { renderToolDisclosureMarkdown } from "../../scripts/bench/tool-disclosure/report"; +import { renderToolDisclosureMarkdown } from "../../scripts/performance/tool-disclosure/report"; import { buildComparisonCells, buildConservativeClaims, @@ -11,16 +11,16 @@ import { evaluateClaimGates, pairedBootstrapDifference, summarizeStaticVisibility, -} from "../../scripts/bench/tool-disclosure/statistics"; +} from "../../scripts/performance/tool-disclosure/statistics"; import { - type BenchmarkPhase, + type PerformanceTestPhase, DEFAULT_BOOTSTRAP_SAMPLES, - TOOL_DISCLOSURE_SCHEMA_VERSION, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, type ToolDisclosureAgent, type ToolDisclosureManifest, type ToolDisclosureMode, type ToolDisclosureRun, -} from "../../scripts/bench/tool-disclosure/types"; +} from "../../scripts/performance/tool-disclosure/types"; const AGENTS = ["langchain-deepagents-code", "hermes", "openclaw"] as const; const CAMPAIGNS = ["campaign-1", "campaign-2"] as const; @@ -28,13 +28,13 @@ const TASK_IDS = Array.from({ length: 24 }, (_, index) => `task-${index + 1}`); function makeManifest(overrides: Partial = {}): ToolDisclosureManifest { return { - schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: "tool-disclosure-example", + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: "tool-disclosure-example", created_at: "2026-07-06T12:00:00.000Z", sut: { git_sha: "3a05b54e", git_ref: "v0.0.74", worktree_clean: true }, harness: { git_sha: "1234abcd", - git_ref: "benchmark-harness", + git_ref: "performance-test-harness", worktree_clean: true, }, campaigns: [ @@ -109,7 +109,7 @@ interface RunOptions { campaign?: string; agent?: ToolDisclosureAgent; mode?: ToolDisclosureMode; - phase?: BenchmarkPhase; + phase?: PerformanceTestPhase; taskId?: string; repetition?: number; success?: boolean; @@ -131,8 +131,8 @@ function makeRun(options: RunOptions = {}): ToolDisclosureRun { const scored = options.scored ?? phase !== "static-visibility"; runSequence += 1; return { - schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: "tool-disclosure-example", + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: "tool-disclosure-example", campaign_id: options.campaign ?? "campaign-1", run_id: `run-${runSequence}`, phase, @@ -423,7 +423,7 @@ describe("tool-disclosure Markdown report", () => { ], }); - expect(markdown).toContain("# Progressive Tool-Disclosure Benchmark"); + expect(markdown).toContain("# Progressive Tool-Disclosure Performance Test"); expect(markdown).toContain("at least 99%"); expect(markdown).toContain("initial serialized tool-schema tokens"); expect(markdown).toContain("not an authorization boundary"); diff --git a/test/bench/tool-disclosure-schedule.test.ts b/test/performance/tool-disclosure-schedule.test.ts similarity index 95% rename from test/bench/tool-disclosure-schedule.test.ts rename to test/performance/tool-disclosure-schedule.test.ts index b003df57ff..70b9a09562 100644 --- a/test/bench/tool-disclosure-schedule.test.ts +++ b/test/performance/tool-disclosure-schedule.test.ts @@ -14,11 +14,11 @@ import { scanArtifactsForForbiddenValues, writeChecksumManifest, writeJsonArtifact, -} from "../../scripts/bench/tool-disclosure/artifacts"; +} from "../../scripts/performance/tool-disclosure/artifacts"; import { buildToolDisclosureSchedule, countScheduledRunsByCampaign, -} from "../../scripts/bench/tool-disclosure/schedule"; +} from "../../scripts/performance/tool-disclosure/schedule"; const primaryTaskIds = Array.from({ length: 24 }, (_, index) => `task-${index + 1}`); const stressTaskIds = primaryTaskIds.slice(0, 8); @@ -56,7 +56,7 @@ describe("tool-disclosure schedule", () => { describe("tool-disclosure artifacts", () => { it("writes JSON, JSONL, checksums, and rejects leaked values", () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-tool-bench-")); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-tool-performance-test-")); try { const output = ensureCampaignDirectory(path.join(root, "campaign"), false); writeJsonArtifact(output, "manifest.json", { safe: true }); diff --git a/test/bench/tool-disclosure-telemetry.test.ts b/test/performance/tool-disclosure-telemetry.test.ts similarity index 85% rename from test/bench/tool-disclosure-telemetry.test.ts rename to test/performance/tool-disclosure-telemetry.test.ts index 0c671a049a..a633226159 100644 --- a/test/bench/tool-disclosure-telemetry.test.ts +++ b/test/performance/tool-disclosure-telemetry.test.ts @@ -10,16 +10,16 @@ import { readVllmProcessStartTime, tokenDelta, validatedTelemetryUrl, -} from "../../scripts/bench/tool-disclosure/telemetry"; +} from "../../scripts/performance/tool-disclosure/telemetry"; -describe("vLLM benchmark telemetry", () => { +describe("vLLM performance test telemetry", () => { it("sums labelled token counters and computes monotonic deltas", () => { const snapshot = parseVllmTokenMetrics(` # HELP vllm:prompt_tokens_total prompt -vllm:prompt_tokens_total{model_name="bench",engine="0"} 100 -vllm:prompt_tokens_total{model_name="bench",engine="1"} 20 -vllm:generation_tokens_total{model_name="bench",engine="0"} 30 -vllm:generation_tokens_total{model_name="bench",engine="1"} 2 +vllm:prompt_tokens_total{model_name="performance-test",engine="0"} 100 +vllm:prompt_tokens_total{model_name="performance-test",engine="1"} 20 +vllm:generation_tokens_total{model_name="performance-test",engine="0"} 30 +vllm:generation_tokens_total{model_name="performance-test",engine="1"} 2 `); expect(snapshot).toEqual({ prompt_tokens: 120, generation_tokens: 32 }); expect(parseProcessStartTime("process_start_time_seconds 1712345678\n")).toBe(1712345678); @@ -49,7 +49,7 @@ vllm:generation_tokens_total{model_name="bench",engine="1"} 2 ); }); - it("rejects non-loopback HTTPS telemetry and tokenizer URLs for the local benchmark", () => { + it("rejects non-loopback HTTPS telemetry and tokenizer URLs for the local performance test", () => { expect(() => validatedTelemetryUrl("https://inference.example/v1", "/metrics")).toThrow( "vLLM telemetry is allowed only on loopback", ); @@ -85,7 +85,7 @@ vllm:generation_tokens_total{model_name="bench",engine="1"} 2 1712345678, ); await expect( - countTokensWithVllm("http://localhost:8000", "bench-model", "[]", tokenizerFetch), + countTokensWithVllm("http://localhost:8000", "performance-test-model", "[]", tokenizerFetch), ).resolves.toBe(4); expect(requestedUrls).toEqual([ "https://127.0.0.1:8443/metrics", diff --git a/test/bench/tool-disclosure-validation.test.ts b/test/performance/tool-disclosure-validation.test.ts similarity index 93% rename from test/bench/tool-disclosure-validation.test.ts rename to test/performance/tool-disclosure-validation.test.ts index 11a0377f2b..c4abd8fdcc 100644 --- a/test/bench/tool-disclosure-validation.test.ts +++ b/test/performance/tool-disclosure-validation.test.ts @@ -3,41 +3,41 @@ import { beforeAll, describe, expect, it } from "vitest"; -import { assembleToolDisclosureRun } from "../../scripts/bench/tool-disclosure/assemble-run"; +import { assembleToolDisclosureRun } from "../../scripts/performance/tool-disclosure/assemble-run"; import { canonicalJson, generateSyntheticCatalog, type SyntheticCatalog, sha256Hex, -} from "../../scripts/bench/tool-disclosure/catalog"; -import type { SanitizedRunEvidence } from "../../scripts/bench/tool-disclosure/execute"; -import type { RecordedSyntheticCall } from "../../scripts/bench/tool-disclosure/grading"; -import type { ToolDisclosureRecordingEvent } from "../../scripts/bench/tool-disclosure/recorder"; +} from "../../scripts/performance/tool-disclosure/catalog"; +import type { SanitizedRunEvidence } from "../../scripts/performance/tool-disclosure/execute"; +import type { RecordedSyntheticCall } from "../../scripts/performance/tool-disclosure/grading"; +import type { ToolDisclosureRecordingEvent } from "../../scripts/performance/tool-disclosure/recorder"; import { buildToolDisclosureSchedule, type ScheduledToolDisclosureRun, STATIC_CATALOG_SIZES, TOOL_DISCLOSURE_AGENTS, TOOL_DISCLOSURE_MODES, -} from "../../scripts/bench/tool-disclosure/schedule"; +} from "../../scripts/performance/tool-disclosure/schedule"; import { generatePrimaryTaskSet, generateStressTaskSet, - type SyntheticBenchmarkTask, + type SyntheticPerformanceTask, type SyntheticTaskSet, -} from "../../scripts/bench/tool-disclosure/tasks"; +} from "../../scripts/performance/tool-disclosure/tasks"; import { DEFAULT_BOOTSTRAP_SAMPLES, DEFAULT_BOOTSTRAP_SEED, DEFAULT_NONINFERIORITY_MARGIN_PP, - TOOL_DISCLOSURE_SCHEMA_VERSION, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, type ToolDisclosureManifest, type ToolDisclosureRun, -} from "../../scripts/bench/tool-disclosure/types"; +} from "../../scripts/performance/tool-disclosure/types"; import { validateCompleteEvidence, validateFrozenManifest, -} from "../../scripts/bench/tool-disclosure/validation"; +} from "../../scripts/performance/tool-disclosure/validation"; interface CompleteEvidenceFixture { catalog: SyntheticCatalog; @@ -65,8 +65,8 @@ function buildManifest( ), ); return { - schema_version: TOOL_DISCLOSURE_SCHEMA_VERSION, - benchmark_id: "tool-disclosure-complete-evidence-fixture", + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: "tool-disclosure-complete-evidence-fixture", created_at: "2026-07-06T12:00:00.000Z", sut: { git_sha: "1".repeat(40), @@ -143,7 +143,7 @@ function buildManifest( }; } -function recordedCalls(task: SyntheticBenchmarkTask | undefined): RecordedSyntheticCall[] { +function recordedCalls(task: SyntheticPerformanceTask | undefined): RecordedSyntheticCall[] { return ( task?.expected_calls.map((call) => ({ tool_name: call.tool_name, From 8204fe1540665377b6a81ae34c561f6ab5b33501 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 20:05:39 -0700 Subject: [PATCH 12/31] fix(performance): preserve tool disclosure in exec shells Signed-off-by: Aaron Erickson --- agents/langchain-deepagents-code/start.sh | 1 + .../tool-disclosure-performance-smoke.test.ts | 2 +- ...ain-deepagents-code-proxy-launcher.test.ts | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/agents/langchain-deepagents-code/start.sh b/agents/langchain-deepagents-code/start.sh index 65cd2a0af0..6b931ad1e2 100755 --- a/agents/langchain-deepagents-code/start.sh +++ b/agents/langchain-deepagents-code/start.sh @@ -165,6 +165,7 @@ prepare_runtime_env() { # LangSmith values are intentionally excluded because any inherited variable # can be misconfigured with a token and this shared file is readable. write_export_if_set NEMOCLAW_SANDBOX_NAME + write_export_if_set NEMOCLAW_TOOL_DISCLOSURE } >"$tmp" # Dcode intentionally runs as the non-root sandbox user, unlike the # root-supervised OpenClaw/Hermes startup path. This atomic, sandbox-user-owned diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 76e4e53388..6ff12c0da1 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -137,7 +137,7 @@ test.skipIf(!shouldRunLiveE2E())( const attestation = await sandbox.exec( name, - ["sh", "-c", `test "$NEMOCLAW_TOOL_DISCLOSURE" = ${JSON.stringify(mode)}`], + ["sh", "-lc", `test "$NEMOCLAW_TOOL_DISCLOSURE" = ${JSON.stringify(mode)}`], { artifactName: `attest-tool-disclosure-${mode}`, env: sandboxAccessEnv(), diff --git a/test/langchain-deepagents-code-proxy-launcher.test.ts b/test/langchain-deepagents-code-proxy-launcher.test.ts index ad20559834..ea7ccfa9e2 100644 --- a/test/langchain-deepagents-code-proxy-launcher.test.ts +++ b/test/langchain-deepagents-code-proxy-launcher.test.ts @@ -198,6 +198,35 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { expect(output).not.toContain("all-password"); }); + it("persists the image-selected tool-disclosure mode for login-shell exec paths", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-disclosure-")); + const { envFile, scriptPath } = makeStartProxyProbeFixture(tempDir); + const startResult = spawnSync("bash", [scriptPath, "/usr/bin/true"], { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + }, + encoding: "utf8", + }); + expect(startResult.status, startResult.stderr).toBe(0); + const envFileText = fs.readFileSync(envFile, "utf8"); + expect(envFileText).toContain("export NEMOCLAW_TOOL_DISCLOSURE=direct"); + + const sourced = spawnSync( + "bash", + ["-c", '. "$1"; printf "%s" "$NEMOCLAW_TOOL_DISCLOSURE"', "bash", envFile], + { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + NEMOCLAW_TOOL_DISCLOSURE: "progressive", + }, + encoding: "utf8", + }, + ); + expect(sourced.status, sourced.stderr).toBe(0); + expect(sourced.stdout).toBe("direct"); + }); + it("pins validated proxy overrides into direct dcode execution paths (#6191)", () => { const dockerfile = readAgentFile("Dockerfile"); const launcher = readAgentFile("dcode-launcher.sh"); From 56464c2891a19625bde6d9aa797bd61a604a70f8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 20:15:26 -0700 Subject: [PATCH 13/31] fix(performance): restore configured disclosure mode Signed-off-by: Aaron Erickson --- agents/langchain-deepagents-code/Dockerfile | 10 +- .../langchain-deepagents-code/Dockerfile.base | 2 +- .../dcode-launcher.sh | 22 +++-- agents/langchain-deepagents-code/start.sh | 26 +++-- test/dcode-start-keepalive.test.ts | 7 ++ .../tool-disclosure-performance-smoke.test.ts | 7 +- .../langchain-deepagents-code-headless.ts | 7 ++ ...ain-deepagents-code-proxy-launcher.test.ts | 94 ++++++++++++++----- test/support/dcode-start-script-fixture.ts | 7 ++ 9 files changed, 138 insertions(+), 44 deletions(-) diff --git a/agents/langchain-deepagents-code/Dockerfile b/agents/langchain-deepagents-code/Dockerfile index 3b2e62774a..9ef5e26243 100644 --- a/agents/langchain-deepagents-code/Dockerfile +++ b/agents/langchain-deepagents-code/Dockerfile @@ -59,14 +59,16 @@ RUN case "$NEMOCLAW_TOOL_DISCLOSURE" in \ esac # The launcher and startup script read these root-owned files instead of -# trusting process-level environment overrides for inference routing. Invoking -# each launcher validates the build args before the image can complete. +# trusting process-level environment overrides for inference routing or tool +# disclosure. Invoking each launcher validates the build args before the image +# can complete. RUN install -d -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' "$NEMOCLAW_PROXY_HOST" > /usr/local/share/nemoclaw/dcode-proxy-host \ && printf '%s\n' "$NEMOCLAW_PROXY_PORT" > /usr/local/share/nemoclaw/dcode-proxy-port \ + && printf '%s\n' "$NEMOCLAW_TOOL_DISCLOSURE" > /usr/local/share/nemoclaw/dcode-tool-disclosure \ && printf '%s\n' "$NEMOCLAW_INFERENCE_BASE_URL" > /usr/local/share/nemoclaw/dcode-inference-base-url \ - && chown root:root /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-inference-base-url \ - && chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-inference-base-url \ + && chown root:root /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-tool-disclosure /usr/local/share/nemoclaw/dcode-inference-base-url \ + && chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-tool-disclosure /usr/local/share/nemoclaw/dcode-inference-base-url \ && /usr/local/bin/dcode --version \ && /usr/local/bin/dcode.real --version \ && /usr/local/bin/deepagents-code --version diff --git a/agents/langchain-deepagents-code/Dockerfile.base b/agents/langchain-deepagents-code/Dockerfile.base index bc9d9883e2..2c6f41fdee 100644 --- a/agents/langchain-deepagents-code/Dockerfile.base +++ b/agents/langchain-deepagents-code/Dockerfile.base @@ -42,7 +42,7 @@ RUN groupadd -r sandbox \ # Pre-create shell init files for the sandbox user. OpenShell/NemoClaw writes # /tmp/nemoclaw-proxy-env.sh at startup so interactive sessions and dcode share -# the same proxy, CA, inference, HOME, and update-check posture. +# the same proxy, CA, inference, tool-disclosure, HOME, and update-check posture. # hadolint ignore=SC2016 RUN printf '%s\n' \ '# Source runtime proxy + Deep Agents Code config' \ diff --git a/agents/langchain-deepagents-code/dcode-launcher.sh b/agents/langchain-deepagents-code/dcode-launcher.sh index 2e92280e62..76c8001707 100755 --- a/agents/langchain-deepagents-code/dcode-launcher.sh +++ b/agents/langchain-deepagents-code/dcode-launcher.sh @@ -20,6 +20,7 @@ export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sb # dcode no longer uses inference.local. readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host" readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port" +readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure" readonly MANAGED_PROXY_OWNER_UID=0 managed_proxy_file_metadata() { @@ -32,21 +33,21 @@ managed_proxy_file_metadata() { fi } -read_managed_proxy_value() { +read_managed_image_value() { local file="$1" local name="$2" local metadata local value if [ ! -f "$file" ] || [ -L "$file" ] || [ ! -r "$file" ]; then - printf 'Missing or unsafe trusted managed proxy %s file.\n' "$name" >&2 + printf 'Missing or unsafe trusted managed %s file.\n' "$name" >&2 return 1 fi metadata="$(managed_proxy_file_metadata "$file")" || { - printf 'Cannot inspect trusted managed proxy %s file.\n' "$name" >&2 + printf 'Cannot inspect trusted managed %s file.\n' "$name" >&2 return 1 } if [ "$metadata" != "${MANAGED_PROXY_OWNER_UID}:444" ]; then - printf 'Unsafe ownership or mode on trusted managed proxy %s file.\n' "$name" >&2 + printf 'Unsafe ownership or mode on trusted managed %s file.\n' "$name" >&2 return 1 fi value="$(<"$file")" @@ -55,8 +56,9 @@ read_managed_proxy_value() { # Onboard validates the build args and the Dockerfile stores them in root-owned # files. Runtime env is untrusted and cannot override those image-baked values. -PROXY_HOST="$(read_managed_proxy_value "$MANAGED_PROXY_HOST_FILE" "host")" -PROXY_PORT="$(read_managed_proxy_value "$MANAGED_PROXY_PORT_FILE" "port")" +PROXY_HOST="$(read_managed_image_value "$MANAGED_PROXY_HOST_FILE" "proxy host")" +PROXY_PORT="$(read_managed_image_value "$MANAGED_PROXY_PORT_FILE" "proxy port")" +TOOL_DISCLOSURE="$(read_managed_image_value "$MANAGED_TOOL_DISCLOSURE_FILE" "tool-disclosure mode")" unset NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT # Generic proxy fallbacks are outside the managed dcode contract and may carry # host credentials even after the scheme-specific proxy values are normalized. @@ -88,6 +90,14 @@ if ! is_valid_proxy_port "$PROXY_PORT"; then printf '%s\n' 'Invalid NEMOCLAW_PROXY_PORT for the managed runtime proxy.' >&2 exit 1 fi +case "$TOOL_DISCLOSURE" in + progressive | direct) ;; + *) + printf '%s\n' 'Invalid image-baked tool-disclosure mode for the managed runtime.' >&2 + exit 1 + ;; +esac +export NEMOCLAW_TOOL_DISCLOSURE="$TOOL_DISCLOSURE" _PROXY_URL="http://${PROXY_HOST}:${PROXY_PORT}" _NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}" diff --git a/agents/langchain-deepagents-code/start.sh b/agents/langchain-deepagents-code/start.sh index 6b931ad1e2..ae19c955e7 100755 --- a/agents/langchain-deepagents-code/start.sh +++ b/agents/langchain-deepagents-code/start.sh @@ -42,6 +42,7 @@ export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference.local/v1}" # or when dcode no longer uses inference.local. readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host" readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port" +readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure" readonly MANAGED_PROXY_OWNER_UID=0 managed_proxy_file_metadata() { @@ -54,31 +55,32 @@ managed_proxy_file_metadata() { fi } -read_managed_proxy_value() { +read_managed_image_value() { local file="$1" local name="$2" local metadata local value if [ ! -f "$file" ] || [ -L "$file" ] || [ ! -r "$file" ]; then - printf 'Missing or unsafe trusted managed proxy %s file.\n' "$name" >&2 + printf 'Missing or unsafe trusted managed %s file.\n' "$name" >&2 return 1 fi metadata="$(managed_proxy_file_metadata "$file")" || { - printf 'Cannot inspect trusted managed proxy %s file.\n' "$name" >&2 + printf 'Cannot inspect trusted managed %s file.\n' "$name" >&2 return 1 } if [ "$metadata" != "${MANAGED_PROXY_OWNER_UID}:444" ]; then - printf 'Unsafe ownership or mode on trusted managed proxy %s file.\n' "$name" >&2 + printf 'Unsafe ownership or mode on trusted managed %s file.\n' "$name" >&2 return 1 fi value="$(<"$file")" printf '%s' "$value" } -# Fail closed if the root-owned image contract is missing. Process-level -# NEMOCLAW_PROXY_* values are not a trusted runtime routing source. -PROXY_HOST="$(read_managed_proxy_value "$MANAGED_PROXY_HOST_FILE" "host")" -PROXY_PORT="$(read_managed_proxy_value "$MANAGED_PROXY_PORT_FILE" "port")" +# Fail closed if the root-owned image contract is missing. Process-level proxy +# and tool-disclosure values are not trusted runtime configuration sources. +PROXY_HOST="$(read_managed_image_value "$MANAGED_PROXY_HOST_FILE" "proxy host")" +PROXY_PORT="$(read_managed_image_value "$MANAGED_PROXY_PORT_FILE" "proxy port")" +TOOL_DISCLOSURE="$(read_managed_image_value "$MANAGED_TOOL_DISCLOSURE_FILE" "tool-disclosure mode")" unset NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT # Generic proxy fallbacks are outside the managed dcode contract and may carry # host credentials even after the scheme-specific proxy values are normalized. @@ -110,6 +112,14 @@ if ! is_valid_proxy_port "$PROXY_PORT"; then printf '%s\n' 'Invalid NEMOCLAW_PROXY_PORT for the managed runtime proxy.' >&2 exit 1 fi +case "$TOOL_DISCLOSURE" in + progressive | direct) ;; + *) + printf '%s\n' 'Invalid image-baked tool-disclosure mode for the managed runtime.' >&2 + exit 1 + ;; +esac +export NEMOCLAW_TOOL_DISCLOSURE="$TOOL_DISCLOSURE" _PROXY_URL="http://${PROXY_HOST}:${PROXY_PORT}" _NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}" diff --git a/test/dcode-start-keepalive.test.ts b/test/dcode-start-keepalive.test.ts index 00412f9c71..9e68e3bdf8 100644 --- a/test/dcode-start-keepalive.test.ts +++ b/test/dcode-start-keepalive.test.ts @@ -24,6 +24,7 @@ function makeStartFixture(): string { const scriptPath = path.join(tempDir, "start.sh"); const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); + const toolDisclosureFile = path.join(tempDir, "trusted-tool-disclosure"); const fixture = fs .readFileSync(START_SCRIPT, "utf8") .replace( @@ -34,14 +35,20 @@ function makeStartFixture(): string { 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', `readonly MANAGED_PROXY_PORT_FILE="${portFile}"`, ) + .replace( + 'readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure"', + `readonly MANAGED_TOOL_DISCLOSURE_FILE="${toolDisclosureFile}"`, + ) .replace( "readonly MANAGED_PROXY_OWNER_UID=0", `readonly MANAGED_PROXY_OWNER_UID=${process.getuid?.() ?? 0}`, ); fs.writeFileSync(hostFile, "10.200.0.1\n"); fs.writeFileSync(portFile, "3128\n"); + fs.writeFileSync(toolDisclosureFile, "progressive\n"); fs.chmodSync(hostFile, 0o444); fs.chmodSync(portFile, 0o444); + fs.chmodSync(toolDisclosureFile, 0o444); fs.writeFileSync(scriptPath, fixture); fs.chmodSync(scriptPath, 0o755); tempDirs.push(tempDir); diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 6ff12c0da1..1096717d02 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -137,7 +137,12 @@ test.skipIf(!shouldRunLiveE2E())( const attestation = await sandbox.exec( name, - ["sh", "-lc", `test "$NEMOCLAW_TOOL_DISCLOSURE" = ${JSON.stringify(mode)}`], + [ + "nemoclaw-start", + "sh", + "-c", + `test "$NEMOCLAW_TOOL_DISCLOSURE" = ${JSON.stringify(mode)}`, + ], { artifactName: `attest-tool-disclosure-${mode}`, env: sandboxAccessEnv(), diff --git a/test/helpers/langchain-deepagents-code-headless.ts b/test/helpers/langchain-deepagents-code-headless.ts index b11a299403..02f6a2f1e9 100644 --- a/test/helpers/langchain-deepagents-code-headless.ts +++ b/test/helpers/langchain-deepagents-code-headless.ts @@ -50,6 +50,7 @@ export function makeStartScriptFixture( const scriptPath = path.join(tempDir, "start.sh"); const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); + const toolDisclosureFile = path.join(tempDir, "trusted-tool-disclosure"); expect(original).toContain("local target=/tmp/nemoclaw-proxy-env.sh"); expect(original).toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); const fixture = original @@ -61,6 +62,10 @@ export function makeStartScriptFixture( 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', `readonly MANAGED_PROXY_PORT_FILE="${portFile}"`, ) + .replace( + 'readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure"', + `readonly MANAGED_TOOL_DISCLOSURE_FILE="${toolDisclosureFile}"`, + ) .replace( "readonly MANAGED_PROXY_OWNER_UID=0", `readonly MANAGED_PROXY_OWNER_UID=${process.getuid?.() ?? 0}`, @@ -76,8 +81,10 @@ export function makeStartScriptFixture( expect(fixture).not.toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); fs.writeFileSync(hostFile, "10.200.0.1\n", "utf8"); fs.writeFileSync(portFile, "3128\n", "utf8"); + fs.writeFileSync(toolDisclosureFile, "progressive\n", "utf8"); fs.chmodSync(hostFile, 0o444); fs.chmodSync(portFile, 0o444); + fs.chmodSync(toolDisclosureFile, 0o444); fs.writeFileSync(scriptPath, fixture, "utf8"); fs.chmodSync(scriptPath, 0o755); return { envFile, scriptPath }; diff --git a/test/langchain-deepagents-code-proxy-launcher.test.ts b/test/langchain-deepagents-code-proxy-launcher.test.ts index ea7ccfa9e2..f7ee1ff1d3 100644 --- a/test/langchain-deepagents-code-proxy-launcher.test.ts +++ b/test/langchain-deepagents-code-proxy-launcher.test.ts @@ -31,15 +31,20 @@ function readAgentFile(name: string): string { function writeManagedProxyFiles( tempDir: string, managedProxy: { host: string; port: string }, + toolDisclosure: "progressive" | "direct" = "progressive", ): void { const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); + const toolDisclosureFile = path.join(tempDir, "trusted-tool-disclosure"); fs.rmSync(hostFile, { force: true }); fs.rmSync(portFile, { force: true }); + fs.rmSync(toolDisclosureFile, { force: true }); fs.writeFileSync(hostFile, `${managedProxy.host}\n`); fs.writeFileSync(portFile, `${managedProxy.port}\n`); + fs.writeFileSync(toolDisclosureFile, `${toolDisclosure}\n`); fs.chmodSync(hostFile, 0o444); fs.chmodSync(portFile, 0o444); + fs.chmodSync(toolDisclosureFile, 0o444); } function replaceManagedProxyFileConstants(source: string, tempDir: string): string { @@ -52,6 +57,10 @@ function replaceManagedProxyFileConstants(source: string, tempDir: string): stri 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', `readonly MANAGED_PROXY_PORT_FILE="${path.join(tempDir, "trusted-proxy-port")}"`, ) + .replace( + 'readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure"', + `readonly MANAGED_TOOL_DISCLOSURE_FILE="${path.join(tempDir, "trusted-tool-disclosure")}"`, + ) .replace( "readonly MANAGED_PROXY_OWNER_UID=0", `readonly MANAGED_PROXY_OWNER_UID=${TEST_OWNER_UID}`, @@ -61,12 +70,13 @@ function replaceManagedProxyFileConstants(source: string, tempDir: string): stri function makeLauncherProxyProbeFixture( tempDir: string, managedProxy: { host: string; port: string } = DEFAULT_MANAGED_PROXY, + toolDisclosure: "progressive" | "direct" = "progressive", ): string { const launcherPath = path.join(tempDir, "dcode-launcher.sh"); const probePath = path.join(tempDir, "managed-dcode-probe.sh"); const probe = [ "#!/bin/bash -p", - "for name in HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ALL_PROXY all_proxy OPENAI_PROXY NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT; do", + "for name in HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ALL_PROXY all_proxy OPENAI_PROXY NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT NEMOCLAW_TOOL_DISCLOSURE; do", ' printf \'LAUNCHER_%s=%s\\n\' "$name" "${!name-__unset__}"', "done", "", @@ -80,7 +90,7 @@ function makeLauncherProxyProbeFixture( ); fs.writeFileSync(probePath, probe, "utf8"); fs.writeFileSync(launcherPath, fixture, "utf8"); - writeManagedProxyFiles(tempDir, managedProxy); + writeManagedProxyFiles(tempDir, managedProxy, toolDisclosure); fs.chmodSync(probePath, 0o755); fs.chmodSync(launcherPath, 0o755); return launcherPath; @@ -89,6 +99,7 @@ function makeLauncherProxyProbeFixture( function makeStartProxyProbeFixture( tempDir: string, managedProxy: { host: string; port: string } = DEFAULT_MANAGED_PROXY, + toolDisclosure: "progressive" | "direct" = "progressive", ): { envFile: string; scriptPath: string } { const envFile = path.join(tempDir, "proxy-env.sh"); const scriptPath = path.join(tempDir, "start.sh"); @@ -99,7 +110,7 @@ function makeStartProxyProbeFixture( `tmp="$(mktemp "${tempDir}/nemoclaw-proxy-env.XXXXXX")"`, ); fs.writeFileSync(scriptPath, fixture, "utf8"); - writeManagedProxyFiles(tempDir, managedProxy); + writeManagedProxyFiles(tempDir, managedProxy, toolDisclosure); fs.chmodSync(scriptPath, 0o755); return { envFile, scriptPath }; } @@ -198,33 +209,35 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { expect(output).not.toContain("all-password"); }); - it("persists the image-selected tool-disclosure mode for login-shell exec paths", () => { + it("restores the image-selected tool-disclosure mode on raw OpenShell exec paths", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-disclosure-")); - const { envFile, scriptPath } = makeStartProxyProbeFixture(tempDir); - const startResult = spawnSync("bash", [scriptPath, "/usr/bin/true"], { - env: { - PATH: process.env.PATH ?? "/usr/bin:/bin", - NEMOCLAW_TOOL_DISCLOSURE: "direct", - }, - encoding: "utf8", - }); - expect(startResult.status, startResult.stderr).toBe(0); - const envFileText = fs.readFileSync(envFile, "utf8"); - expect(envFileText).toContain("export NEMOCLAW_TOOL_DISCLOSURE=direct"); - - const sourced = spawnSync( + const launcherPath = makeLauncherProxyProbeFixture(tempDir, DEFAULT_MANAGED_PROXY, "direct"); + const { envFile, scriptPath } = makeStartProxyProbeFixture( + tempDir, + DEFAULT_MANAGED_PROXY, + "direct", + ); + const untrustedEnv = { NEMOCLAW_TOOL_DISCLOSURE: "progressive" }; + const launcherResult = runLauncher(launcherPath, ["-n", "PONG"], untrustedEnv); + const startResult = spawnSync( "bash", - ["-c", '. "$1"; printf "%s" "$NEMOCLAW_TOOL_DISCLOSURE"', "bash", envFile], + [ + scriptPath, + "bash", + "-c", + 'printf "START_TOOL_DISCLOSURE=%s\\n" "$NEMOCLAW_TOOL_DISCLOSURE"', + ], { - env: { - PATH: process.env.PATH ?? "/usr/bin:/bin", - NEMOCLAW_TOOL_DISCLOSURE: "progressive", - }, + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", ...untrustedEnv }, encoding: "utf8", }, ); - expect(sourced.status, sourced.stderr).toBe(0); - expect(sourced.stdout).toBe("direct"); + + expect(launcherResult.status, launcherResult.stderr).toBe(0); + expect(startResult.status, startResult.stderr).toBe(0); + expect(launcherResult.stdout).toContain("LAUNCHER_NEMOCLAW_TOOL_DISCLOSURE=direct"); + expect(startResult.stdout).toContain("START_TOOL_DISCLOSURE=direct"); + expect(fs.readFileSync(envFile, "utf8")).toContain("export NEMOCLAW_TOOL_DISCLOSURE=direct"); }); it("pins validated proxy overrides into direct dcode execution paths (#6191)", () => { @@ -237,6 +250,11 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { expect(dockerfile).toContain("printf '%s\\n' \"$NEMOCLAW_PROXY_PORT\""); expect(dockerfile).toContain("chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host"); expect(dockerfile).toContain("chown root:root /usr/local/share/nemoclaw/dcode-proxy-host"); + expect(dockerfile).toContain( + "printf '%s\\n' \"$NEMOCLAW_TOOL_DISCLOSURE\" > /usr/local/share/nemoclaw/dcode-tool-disclosure", + ); + expect(dockerfile).toMatch(/chmod 0444 .*dcode-tool-disclosure/u); + expect(dockerfile).toMatch(/chown root:root .*dcode-tool-disclosure/u); expect(dockerfile).not.toContain(" NEMOCLAW_PROXY_HOST=${NEMOCLAW_PROXY_HOST}"); expect(dockerfile).not.toContain(" NEMOCLAW_PROXY_PORT=${NEMOCLAW_PROXY_PORT}"); expect(launcher).toContain('readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw'); @@ -354,6 +372,34 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { ); }); + it("fails closed on missing or invalid image-baked tool-disclosure state", () => { + for (const invalidValue of [null, "unexpected"] as const) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-disclosure-invalid-")); + const launcherPath = makeLauncherProxyProbeFixture(tempDir); + const { scriptPath } = makeStartProxyProbeFixture(tempDir); + const modeFile = path.join(tempDir, "trusted-tool-disclosure"); + fs.rmSync(modeFile); + if (invalidValue !== null) { + fs.writeFileSync(modeFile, `${invalidValue}\n`, { mode: 0o444 }); + } + + const env = { NEMOCLAW_TOOL_DISCLOSURE: "direct" }; + const launcherResult = runLauncher(launcherPath, ["-n", "PONG"], env); + const startResult = spawnSync("bash", [scriptPath, "true"], { + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", ...env }, + encoding: "utf8", + }); + + expect(launcherResult.status).not.toBe(0); + expect(startResult.status).not.toBe(0); + const combined = `${launcherResult.stdout}\n${launcherResult.stderr}\n${startResult.stdout}\n${startResult.stderr}`; + expect(combined).toMatch( + /trusted managed tool-disclosure mode file|Invalid image-baked tool-disclosure mode/, + ); + expect(combined).not.toContain("LAUNCHER_NEMOCLAW_TOOL_DISCLOSURE=direct"); + } + }); + it("keeps dcode shell proxy validators aligned with onboard validation (#6191)", () => { const start = readAgentFile("start.sh"); const launcher = readAgentFile("dcode-launcher.sh"); diff --git a/test/support/dcode-start-script-fixture.ts b/test/support/dcode-start-script-fixture.ts index 9ba4e6271f..d9c9328d49 100644 --- a/test/support/dcode-start-script-fixture.ts +++ b/test/support/dcode-start-script-fixture.ts @@ -22,6 +22,7 @@ export function makeStartScriptFixture(tempDir: string): { const scriptPath = path.join(tempDir, "start.sh"); const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); + const toolDisclosureFile = path.join(tempDir, "trusted-tool-disclosure"); const original = fs.readFileSync(START_SCRIPT, "utf8"); assert.ok(original.includes("local target=/tmp/nemoclaw-proxy-env.sh")); assert.ok(original.includes('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"')); @@ -34,6 +35,10 @@ export function makeStartScriptFixture(tempDir: string): { 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', `readonly MANAGED_PROXY_PORT_FILE="${portFile}"`, ) + .replace( + 'readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure"', + `readonly MANAGED_TOOL_DISCLOSURE_FILE="${toolDisclosureFile}"`, + ) .replace( "readonly MANAGED_PROXY_OWNER_UID=0", `readonly MANAGED_PROXY_OWNER_UID=${process.getuid?.() ?? 0}`, @@ -49,8 +54,10 @@ export function makeStartScriptFixture(tempDir: string): { assert.ok(!fixture.includes('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"')); fs.writeFileSync(hostFile, "10.200.0.1\n", "utf8"); fs.writeFileSync(portFile, "3128\n", "utf8"); + fs.writeFileSync(toolDisclosureFile, "progressive\n", "utf8"); fs.chmodSync(hostFile, 0o444); fs.chmodSync(portFile, 0o444); + fs.chmodSync(toolDisclosureFile, 0o444); fs.writeFileSync(scriptPath, fixture, "utf8"); fs.chmodSync(scriptPath, 0o755); return { envFile, scriptPath }; From 124b2f4cbc64f59a9e94bd387bd2b3baf7051cc5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 20:21:47 -0700 Subject: [PATCH 14/31] fix(performance): use service-scoped MCP token Signed-off-by: Aaron Erickson --- scripts/performance/tool-disclosure/execute.ts | 2 +- test/e2e/live/tool-disclosure-performance-smoke.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/performance/tool-disclosure/execute.ts b/scripts/performance/tool-disclosure/execute.ts index ca84d4716f..9d2f495e61 100644 --- a/scripts/performance/tool-disclosure/execute.ts +++ b/scripts/performance/tool-disclosure/execute.ts @@ -641,7 +641,7 @@ export async function executeCampaign(options: { existing.filter((run) => run.outcome !== "setup-error").map((run) => run.run_id), ); const token = randomBytes(32).toString("hex"); - const tokenEnv = "NEMOCLAW_PERFORMANCE_TEST_MCP_BEARER"; + const tokenEnv = "TOOL_DISCLOSURE_PERFORMANCE_TEST_MCP_TOKEN"; const proxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl: options.config.upstream_vllm_url, port: options.config.recorder_port, diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 1096717d02..2af313d868 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -29,7 +29,7 @@ const CATALOG_SIZE = 64; const TASK_ID = "primary-single-01"; const MODES = ["progressive", "direct"] as const satisfies readonly ToolDisclosureMode[]; const MCP_SERVER_NAME = "performance-test"; -const MCP_TOKEN_ENV = "NEMOCLAW_PERFORMANCE_TEST_MCP_BEARER"; +const MCP_TOKEN_ENV = "TOOL_DISCLOSURE_PERFORMANCE_TEST_MCP_TOKEN"; const TEST_TIMEOUT_MS = 50 * 60_000; function sandboxName(mode: ToolDisclosureMode): string { From 5cf30fdbbca982db0ec6b85fba8ff6fbbaf3a43a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 20:28:22 -0700 Subject: [PATCH 15/31] fix(performance): keep agent commands single-line Signed-off-by: Aaron Erickson --- scripts/performance/tool-disclosure/drivers.ts | 15 +++++++-------- test/performance/tool-disclosure-drivers.test.ts | 3 ++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/performance/tool-disclosure/drivers.ts b/scripts/performance/tool-disclosure/drivers.ts index 518910b7db..f417e6d921 100644 --- a/scripts/performance/tool-disclosure/drivers.ts +++ b/scripts/performance/tool-disclosure/drivers.ts @@ -31,14 +31,14 @@ function shellScriptFor(agent: ToolDisclosureAgent, prompt: string, sessionId: s "set -eu", `prompt=$(printf '%s' '${encodedPrompt}' | base64 -d)`, `exec openclaw agent --agent main --json --thinking off --session-id '${sessionId}' -m "$prompt"`, - ].join("\n"); + ].join("; "); } if (agent === "langchain-deepagents-code") { return [ "set -eu", `prompt=$(printf '%s' '${encodedPrompt}' | base64 -d)`, 'exec nemoclaw-start dcode -n "$prompt"', - ].join("\n"); + ].join("; "); } const payload = base64( JSON.stringify({ @@ -54,12 +54,8 @@ function shellScriptFor(agent: ToolDisclosureAgent, prompt: string, sessionId: s "[ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env", "set +a", `payload='${payload}'`, - 'if [ -n "${API_SERVER_KEY:-}" ]; then', - " printf '%s' \"$payload\" | base64 -d | curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer ${API_SERVER_KEY}\" --data-binary @-", - " exit $?", - "fi", - "printf '%s' \"$payload\" | base64 -d | curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary @-", - ].join("\n"); + "if [ -n \"${API_SERVER_KEY:-}\" ]; then printf '%s' \"$payload\" | base64 -d | curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer ${API_SERVER_KEY}\" --data-binary @-; else printf '%s' \"$payload\" | base64 -d | curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary @-; fi", + ].join("; "); } export function buildAgentDriverCommand(options: { @@ -73,6 +69,9 @@ export function buildAgentDriverCommand(options: { validateIdentifier(options.sessionId, "sessionId"); if (!options.prompt.trim()) throw new Error("performance test task prompt must not be empty"); const script = shellScriptFor(options.agent, options.prompt, options.sessionId); + if (/[\r\n]/u.test(script)) { + throw new Error("performance test agent driver must be a single-line shell command"); + } return { command: options.openshellBin ?? "openshell", args: ["sandbox", "exec", "-n", options.sandboxName, "--", "sh", "-lc", script], diff --git a/test/performance/tool-disclosure-drivers.test.ts b/test/performance/tool-disclosure-drivers.test.ts index 87625259b0..dca380c5bd 100644 --- a/test/performance/tool-disclosure-drivers.test.ts +++ b/test/performance/tool-disclosure-drivers.test.ts @@ -17,7 +17,7 @@ describe("tool-disclosure agent drivers", () => { "hermes", "langchain-deepagents-code", ] as const)("builds a bounded sandbox command for %s", (agent) => { - const prompt = "Use the weather converter and return PERFORMANCE_TEST_OK_123"; + const prompt = "Use the weather converter,\nthen return PERFORMANCE_TEST_OK_123"; const command = buildAgentDriverCommand({ agent, sandboxName: "performance-test-openclaw-progressive-512", @@ -33,6 +33,7 @@ describe("tool-disclosure agent drivers", () => { "--", ]); expect(command.args.join(" ")).not.toContain(prompt); + expect(command.args.every((argument) => !/[\r\n]/u.test(argument))).toBe(true); expect(command.redactions).toContain(prompt); }); From 2ce142748dcbada5d9d2fa65b96884afeafe648e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 6 Jul 2026 20:42:09 -0700 Subject: [PATCH 16/31] test: track the fourth cloudflared pin Signed-off-by: Aaron Erickson --- scripts/checks/check-cloudflared-update.sh | 18 +++++++++--------- test/cloudflared-update-check-workflow.test.ts | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/checks/check-cloudflared-update.sh b/scripts/checks/check-cloudflared-update.sh index 3f25aa2fc6..8d99d38a41 100755 --- a/scripts/checks/check-cloudflared-update.sh +++ b/scripts/checks/check-cloudflared-update.sh @@ -2,14 +2,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# invalidState: the three reviewed E2E consumers drift to different cloudflared +# invalidState: the four reviewed E2E consumers drift to different cloudflared # versions/digests, or their shared pin no longer matches the upstream asset. -# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all three +# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all four # workflow pins and independently verifies the downloaded bytes. # whyNotSourceFix: upstream cannot enforce which release NemoClaw workflows use. -# regressionTest: cloudflared-update-check-workflow.test.ts covers three-pin +# regressionTest: cloudflared-update-check-workflow.test.ts covers four-pin # parity, asset URL identity, digest mismatch, and update instructions. -# removalCondition: remove this checker when the three consumers share one +# removalCondition: remove this checker when the four consumers share one # machine-readable dependency manifest with equivalent live asset verification. set -euo pipefail @@ -48,10 +48,10 @@ done < <( "${E2E_WORKFLOW}" ) -[[ "${#version_pins[@]}" -eq 3 ]] \ - || fail "expected exactly three CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}" -[[ "${#sha_pins[@]}" -eq 3 ]] \ - || fail "expected exactly three CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}" +[[ "${#version_pins[@]}" -eq 4 ]] \ + || fail "expected exactly four CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}" +[[ "${#sha_pins[@]}" -eq 4 ]] \ + || fail "expected exactly four CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}" pinned_version="${version_pins[0]}" pinned_sha="$(printf '%s' "${sha_pins[0]}" | tr '[:upper:]' '[:lower:]')" @@ -127,7 +127,7 @@ print_update_instructions() { 'Update locations:' \ " ${workflow_display} CLOUDFLARED_VERSION lines: ${version_lines}" \ " ${workflow_display} CLOUDFLARED_DEB_SHA256 lines: ${sha_lines}" \ - 'Set all three version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2 + 'Set all four version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2 } if [[ "${latest_version}" != "${pinned_version}" ]]; then diff --git a/test/cloudflared-update-check-workflow.test.ts b/test/cloudflared-update-check-workflow.test.ts index 99ee32e346..65635b76c9 100644 --- a/test/cloudflared-update-check-workflow.test.ts +++ b/test/cloudflared-update-check-workflow.test.ts @@ -40,7 +40,7 @@ function pinValues(source: string, name: string): string[] { function writePinFixture(file: string, version: string, sha256: string): void { fs.writeFileSync( file, - ["one", "two", "three"] + ["one", "two", "three", "four"] .map( (job) => ` ${job}:\n env:\n CLOUDFLARED_VERSION: "${version}"\n CLOUDFLARED_DEB_SHA256: "${sha256}"`, @@ -140,11 +140,11 @@ describe("cloudflared update-check workflow contract", () => { expect(check?.run).toBe("bash scripts/checks/check-cloudflared-update.sh"); }); - it("extracts exactly three identical reviewed version and SHA256 pins", () => { + it("extracts exactly four identical reviewed version and SHA256 pins", () => { const versions = pinValues(e2e, "CLOUDFLARED_VERSION"); const hashes = pinValues(e2e, "CLOUDFLARED_DEB_SHA256"); - expect(versions).toHaveLength(3); - expect(hashes).toHaveLength(3); + expect(versions).toHaveLength(4); + expect(hashes).toHaveLength(4); expect(new Set(versions).size).toBe(1); expect(new Set(hashes).size).toBe(1); expect(versions[0]).toMatch(/^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$/u); @@ -188,7 +188,7 @@ describe("cloudflared update-check workflow contract", () => { ); expect(fixture.result.stderr).toContain("CLOUDFLARED_VERSION lines:"); expect(fixture.result.stderr).toContain("CLOUDFLARED_DEB_SHA256 lines:"); - expect(fixture.result.stderr).toContain("Set all three version/SHA256 pairs"); + expect(fixture.result.stderr).toContain("Set all four version/SHA256 pairs"); } finally { fs.rmSync(fixture.tempDir, { recursive: true, force: true }); } From 1fc162c951709554d281623e5805ec3fc3ea36f3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 08:35:39 -0700 Subject: [PATCH 17/31] feat(performance): add compositional tool routing acceptance Signed-off-by: Aaron Erickson --- ...ssive-tool-disclosure-performance-test.mdx | 80 +- scripts/performance/README.md | 35 + .../compositional-tool-router.ts | 697 ++++++++++++++++++ .../compositional-tool-routing-acceptance.ts | 649 ++++++++++++++++ .../compositional-tool-routing-adapters.ts | 375 ++++++++++ .../compositional-tool-routing-run.ts | 309 ++++++++ .../compositional-tool-routing-transform.ts | 349 +++++++++ .../performance/tool-disclosure/recorder.ts | 119 ++- scripts/performance/tool-disclosure/run.ts | 41 +- .../tool-disclosure-performance-smoke.test.ts | 188 +++++ ...ol-disclosure-compositional-router.test.ts | 411 +++++++++++ ...e-compositional-routing-acceptance.test.ts | 353 +++++++++ ...ure-compositional-routing-adapters.test.ts | 163 ++++ ...sclosure-compositional-routing-run.test.ts | 155 ++++ ...re-compositional-routing-transform.test.ts | 407 ++++++++++ .../tool-disclosure-recorder.test.ts | 272 +++++++ 16 files changed, 4586 insertions(+), 17 deletions(-) create mode 100644 scripts/performance/tool-disclosure/compositional-tool-router.ts create mode 100644 scripts/performance/tool-disclosure/compositional-tool-routing-acceptance.ts create mode 100644 scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts create mode 100644 scripts/performance/tool-disclosure/compositional-tool-routing-run.ts create mode 100644 scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts create mode 100644 test/performance/tool-disclosure-compositional-router.test.ts create mode 100644 test/performance/tool-disclosure-compositional-routing-acceptance.test.ts create mode 100644 test/performance/tool-disclosure-compositional-routing-adapters.test.ts create mode 100644 test/performance/tool-disclosure-compositional-routing-run.test.ts create mode 100644 test/performance/tool-disclosure-compositional-routing-transform.test.ts diff --git a/docs/inference/progressive-tool-disclosure-performance-test.mdx b/docs/inference/progressive-tool-disclosure-performance-test.mdx index 45638718b3..85efe97451 100644 --- a/docs/inference/progressive-tool-disclosure-performance-test.mdx +++ b/docs/inference/progressive-tool-disclosure-performance-test.mdx @@ -18,9 +18,87 @@ Run both campaigns and retain their evidence before you make a performance or qu The explicit-only `tool-disclosure-performance-smoke` E2E job is a smaller operational check for ordinary GitHub-hosted runners. -It runs one frozen task in progressive and direct modes with fresh Deep Agents sandboxes and hosted inference. +It runs the route-only acceptance corpus, one frozen task in progressive and direct modes with fresh Deep Agents sandboxes, and a separate routed replay of that task through the request transformer. Its output validates live wiring only: it is not a complete campaign, cannot be summarized by this protocol, and does not support performance claims. +## Exercise the compositional routing experiment + +The performance-test package also contains a test-scoped, independently implemented compositional tool router. +It is an experimental reference path, not a new NemoClaw runtime mode. +Enabling it in tests does not change the meaning of the existing `direct` or `progressive` modes. + +The implementation performs these steps: + +1. Ask an injected decomposer for an ordered list of atomic actions. +2. L2-normalize tool and action embeddings, then perform deterministic exact inner-product retrieval. +3. Merge the first-pass candidates into at most `12` tool-name hints. +4. Ask the decomposer to refine the action boundaries with those hints. +5. Retrieve the top `10` candidates per refined action, select a bounded ordered tool set, and filter only the schemas forwarded in a copied model request. + +The tested request-transform contract leaves the caller-owned executor and policy registry unchanged. +Its integration tests verify that only the copied model request is filtered and that the recording proxy measures the transformed schemas. +This transform is not installed in the frozen direct/progressive campaign. +Malformed decomposition, embedding, or routing results fail open to the original unfiltered request, and the route records counts, safe tool names, fallback class, and timings without retaining prompts or subtask text. + +The model adapters support an OpenAI-compatible chat decomposer and an OpenAI-compatible semantic embeddings endpoint. +Record the exact embedding model and revision for any measured run. +The dependency-free hashing adapter is only for deterministic tests and portable smoke checks; it is lexical and is not a substitute for a recorded semantic retrieval model. + +Run the deterministic routing acceptance suite on an ordinary CPU runner: + +```bash +npx vitest run --project integration \ + test/performance/tool-disclosure-compositional-*.test.ts \ + test/performance/tool-disclosure-recorder.test.ts +``` + +Run the same route-only acceptance corpus with a real OpenAI-compatible decomposer by creating a private configuration file: + +```json +{ + "decomposer": { + "base_url": "https://models.example/v1", + "model": "recorded-decomposer-model", + "revision": "immutable-model-revision", + "api_key_env": "MODEL_API_KEY", + "allow_remote": true + }, + "embedding": { + "kind": "portable", + "dimensions": 1024 + } +} +``` + +Then run: + +```bash +npm run performance:tool-disclosure -- route-acceptance \ + --output-dir \ + --config +``` + +The command writes `compositional-routing-acceptance.json` and `SHA256SUMS`, exits nonzero when a strict gate fails, and never writes endpoint URLs, credentials, prompts, or decomposed subtask text. +The portable embedding mode is a CPU smoke path and is always claim-ineligible. +For a semantic retrieval run, set `embedding.kind` to `openai` and provide its `base_url`, `model`, immutable `revision`, optional `api_key_env`, and remote opt-in. +Remote opt-in sends the acceptance prompts and synthetic tool metadata to the configured HTTPS services; HTTP is accepted only on loopback. + +The route-only corpus contains `20` distinct tools and eight implicit requests spanning zero through five required capabilities. +Its strict gates require every route to complete without fallback, exact decomposition count, decomposition within one step, exact tool recall at `1` and `10`, ordered-chain forwarding, forwarded-schema count, and no-tool behavior. +Passthrough is evaluated as forwarding all `20` schemas, so a failed-open no-tool route cannot be scored as a successful empty route. +The deterministic suite holds decomposition fixed, so passing it proves the router, retriever, transformer, evidence, and fallback contracts; it does not prove that tool hints improve a real model's decomposition. + +For a model-level evaluation, record a paired initial-versus-one-refinement run on the same implicit corpus and report decomposition and embedding usage separately alongside decomposition accuracy, decomposition within one step, exact recall at `1` and `10`, ordered-chain selection, selected-schema count, tokens, and latency. +Then use the existing agent performance test below for execution success, arguments, ordering, schema tokens, and end-to-end latency. +Treat latency as a measurement rather than an acceptance gate because the refinement adds a second decomposition call. + +This experiment routes callable tool schemas in its own synthetic corpus. +It does not implement or claim a dependency-graph planner or compatibility composer. +The route-only command does not execute an agent or a tool; use the full performance procedure below to validate task execution and end-to-end behavior. +It also does not install the router into the frozen direct/progressive campaign. +The opt-in smoke job exercises the transform in one separate, claim-ineligible agent replay and requires successful task execution, the expected tool call, no routing fallback, and fewer model-visible schemas. +A broader agent-level routing comparison must use a separate initial/refined schedule in which both agent cells expose the same complete catalog before the request transformer runs. + ## Understand the frozen protocol The preparation command generates one deterministic catalog, task corpus, and paired execution schedule. diff --git a/scripts/performance/README.md b/scripts/performance/README.md index e3a2be971f..a26843a2d4 100644 --- a/scripts/performance/README.md +++ b/scripts/performance/README.md @@ -57,6 +57,41 @@ The summarizer writes `summary.json`, `report.md`, `evidence.json`, and blocked unless the required confidence-interval gate passes for every agent in both campaigns. +The same package includes a test-scoped independent compositional tool routing +experiment. It runs two-pass atomic decomposition, normalized exact +inner-product retrieval, and a bounded tool-hint refinement. Integration tests +exercise its request-transform contract through the recorder while leaving the +caller-owned executor registry unchanged. The transform is not enabled in the +frozen direct/progressive campaign. Run its deterministic CPU acceptance suite +with: + +```bash +npx vitest run --project integration \ + test/performance/tool-disclosure-compositional-*.test.ts \ + test/performance/tool-disclosure-recorder.test.ts +``` + +Those tests prove routing mechanics and strict route-quality gates with frozen +decomposition inputs. They do not establish a model-level decomposition +improvement. The full protocol explains the separate paired initial/refined +evidence required for that result. + +The explicit live smoke also runs a separate routed replay after its frozen +direct/progressive cells. That replay must preserve task correctness and the +expected tool call while reducing model-visible schemas without fallback. It +does not alter either frozen cell and remains claim-ineligible. + +Run the route-only corpus with a real decomposer using: + +```bash +npm run performance:tool-disclosure -- route-acceptance \ + --output-dir \ + --config +``` + +The protocol documents the private config, semantic and portable embedding +options, public-safe output, remote-content boundary, and strict exit gates. + See the [progressive tool-disclosure performance-test protocol](../../docs/inference/progressive-tool-disclosure-performance-test.mdx) for the hardware-neutral workflow, recorder topology, artifact rules, claim gates, and limitations. diff --git a/scripts/performance/tool-disclosure/compositional-tool-router.ts b/scripts/performance/tool-disclosure/compositional-tool-router.ts new file mode 100644 index 0000000000..84cf274330 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-router.ts @@ -0,0 +1,697 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { performance } from "node:perf_hooks"; + +/** + * A small, independent compositional tool router for the performance-test + * harness. Model prompting and text embedding are injected so the routing + * mechanics remain provider- and hardware-neutral. + */ + +export const COMPOSITIONAL_ROUTER_TOP_K = 10; +export const COMPOSITIONAL_ROUTER_HINT_COUNT = 12; +export const COMPOSITIONAL_ROUTER_MAX_SELECTED_TOOLS = 16; +export const COMPOSITIONAL_ROUTER_MAX_SUBTASKS = 12; +export const COMPOSITIONAL_ROUTER_MAX_SUBTASK_CHARS = 512; + +export type DecompositionPass = "initial" | "refined"; + +export interface DecompositionRequest { + pass: DecompositionPass; + query: string; + /** Empty for the initial pass and populated only for the refinement pass. */ + tool_hints: readonly string[]; + signal?: AbortSignal; +} + +export interface TaskDecomposer { + /** Implementations return a JSON-like array; the router validates it at runtime. */ + decompose(request: DecompositionRequest): Promise; +} + +export interface TextEmbedder { + /** + * Return one dense vector per input string. The router normalizes every + * vector before exact inner-product search. + */ + embed(texts: readonly string[], signal?: AbortSignal): Promise; +} + +export interface ToolRoutingCatalogEntry { + name: string; + description: string; + /** Kept opaque and never copied into routing evidence. */ + definition: Definition; +} + +export interface DenseToolVector { + name: string; + vector: readonly number[]; +} + +export interface DenseToolIndex { + dimension: number; + tools: readonly DenseToolVector[]; +} + +export interface RankedTool { + name: string; + score: number; +} + +export type RoutingFallbackReason = + | "invalid-query" + | "invalid-catalog" + | "initial-decomposition-failed" + | "initial-decomposition-malformed" + | "catalog-embedding-failed" + | "subtask-embedding-failed" + | "no-candidates" + | "refinement-failed" + | "refinement-malformed" + | "refinement-no-tool-disagreement" + | "selection-limit-exceeded"; + +export interface CompositionalRoutingTimings { + initial_decomposition_ms: number; + catalog_embedding_ms: number; + initial_retrieval_ms: number; + refinement_ms: number; + final_retrieval_ms: number; + total_ms: number; +} + +/** Prompt-free evidence. Exact tool names require a reviewed public catalog. */ +export interface CompositionalRoutingEvidence { + initial_subtask_count: number; + refined_subtask_count: number; + decomposition_passes: number; + hint_count: number; + hint_tool_names: readonly string[]; + initial_candidate_counts: readonly number[]; + initial_candidate_tool_names: readonly (readonly string[])[]; + final_candidate_counts: readonly number[]; + final_candidate_tool_names: readonly (readonly string[])[]; + selected_tool_count: number; + selected_tool_names: readonly string[]; + fallback: RoutingFallbackReason | null; + timings: CompositionalRoutingTimings; +} + +export interface CompositionalRoutingResult { + /** `passthrough` tells the caller to preserve its complete unfiltered catalog. */ + disposition: "routed" | "passthrough"; + selected_tool_names: readonly string[]; + evidence: CompositionalRoutingEvidence; +} + +export interface CompositionalRouterOptions { + decomposer: TaskDecomposer; + embedder: TextEmbedder; + /** Zero runs the initial route; one adds a tool-informed refinement pass. */ + refinementPasses?: 0 | 1; + topK?: number; + hintCount?: number; + maxSelectedTools?: number; + /** Reuse a prebuilt normalized index instead of embedding the catalog per query. */ + preparedIndex?: DenseToolIndex | Promise; + signal?: AbortSignal; + clock?: () => number; +} + +export interface PairedCompositionalRoutingResult { + initial: CompositionalRoutingResult; + refined: CompositionalRoutingResult; + shared_initial_decomposition: true; + shared_initial_subtask_count: number; +} + +interface MutableEvidence { + initialSubtaskCount: number; + refinedSubtaskCount: number; + decompositionPasses: number; + hintNames: string[]; + initialCandidates: RankedTool[][]; + finalCandidates: RankedTool[][]; + selectedNames: string[]; + timings: CompositionalRoutingTimings; +} + +class RoutingFailure extends Error { + constructor(readonly reason: RoutingFallbackReason) { + super(reason); + } +} + +function compareNames(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + +function validatePositiveInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new TypeError(`${label} must be a positive safe integer`); + } +} + +function roundedMilliseconds(value: number): number { + return Number(Math.max(0, value).toFixed(3)); +} + +function elapsed(clock: () => number, startedAt: number): number { + return roundedMilliseconds(clock() - startedAt); +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new DOMException("routing aborted", "AbortError"); +} + +/** Normalize a finite, non-zero dense vector to unit L2 length. */ +export function l2Normalize(vector: readonly number[]): number[] { + if (!Array.isArray(vector) || vector.length === 0) { + throw new TypeError("embedding vectors must be non-empty arrays"); + } + let squaredNorm = 0; + for (const value of vector) { + if (!Number.isFinite(value)) throw new TypeError("embedding vectors must be finite"); + squaredNorm += value * value; + } + if (!Number.isFinite(squaredNorm) || squaredNorm <= 0) { + throw new TypeError("embedding vectors must have a positive finite L2 norm"); + } + const norm = Math.sqrt(squaredNorm); + return vector.map((value) => value / norm); +} + +function exactInnerProduct(left: readonly number[], right: readonly number[]): number { + if (left.length !== right.length) throw new TypeError("embedding dimensions do not match"); + let score = 0; + for (let index = 0; index < left.length; index += 1) score += left[index] * right[index]; + return Object.is(score, -0) ? 0 : score; +} + +function validateCatalog( + catalog: readonly ToolRoutingCatalogEntry[], +): void { + if (!Array.isArray(catalog)) throw new RoutingFailure("invalid-catalog"); + const names = new Set(); + for (const entry of catalog) { + if ( + !entry || + typeof entry.name !== "string" || + !entry.name.trim() || + entry.name !== entry.name.trim() || + typeof entry.description !== "string" || + names.has(entry.name) + ) { + throw new RoutingFailure("invalid-catalog"); + } + names.add(entry.name); + } +} + +function toolText(entry: ToolRoutingCatalogEntry): string { + return `${entry.name}\n${entry.description}`; +} + +function normalizedMatrix( + vectors: readonly (readonly number[])[], + expectedRows: number, +): { dimension: number; vectors: number[][] } { + if (!Array.isArray(vectors) || vectors.length !== expectedRows) { + throw new TypeError("embedder returned the wrong number of vectors"); + } + if (expectedRows === 0) return { dimension: 0, vectors: [] }; + const normalized = vectors.map((vector) => l2Normalize(vector)); + const dimension = normalized[0].length; + if (normalized.some((vector) => vector.length !== dimension)) { + throw new TypeError("embedder returned inconsistent vector dimensions"); + } + return { dimension, vectors: normalized }; +} + +/** Build a normalized dense index without retaining opaque tool definitions. */ +export async function buildDenseToolIndex( + catalog: readonly ToolRoutingCatalogEntry[], + embedder: TextEmbedder, + signal?: AbortSignal, +): Promise { + validateCatalog(catalog); + if (catalog.length === 0) return { dimension: 0, tools: [] }; + throwIfAborted(signal); + const embedded = await embedder.embed(catalog.map(toolText), signal); + throwIfAborted(signal); + const matrix = normalizedMatrix(embedded, catalog.length); + return { + dimension: matrix.dimension, + tools: catalog.map((entry, index) => ({ + name: entry.name, + vector: matrix.vectors[index], + })), + }; +} + +/** Rank a normalized exact inner-product index with deterministic name ties. */ +export function exactInnerProductTopK( + index: DenseToolIndex, + queryVector: readonly number[], + k = COMPOSITIONAL_ROUTER_TOP_K, +): RankedTool[] { + validatePositiveInteger(k, "k"); + if (index.tools.length === 0) return []; + const normalizedQuery = l2Normalize(queryVector); + if (normalizedQuery.length !== index.dimension) { + throw new TypeError("query and tool embedding dimensions do not match"); + } + return index.tools + .map((tool) => ({ + name: tool.name, + score: exactInnerProduct(normalizedQuery, tool.vector), + })) + .sort((left, right) => right.score - left.score || compareNames(left.name, right.name)) + .slice(0, k); +} + +/** Merge per-subtask candidates by maximum score, then apply a deterministic cap. */ +export function unionToolHints( + candidatesBySubtask: readonly (readonly RankedTool[])[], + limit = COMPOSITIONAL_ROUTER_HINT_COUNT, +): RankedTool[] { + validatePositiveInteger(limit, "hint limit"); + const maximumScores = new Map(); + for (const candidates of candidatesBySubtask) { + for (const candidate of candidates) { + if (!candidate.name || !Number.isFinite(candidate.score)) continue; + const current = maximumScores.get(candidate.name); + if (current === undefined || candidate.score > current) { + maximumScores.set(candidate.name, candidate.score); + } + } + } + return [...maximumScores.entries()] + .map(([name, score]) => ({ name, score })) + .sort((left, right) => right.score - left.score || compareNames(left.name, right.name)) + .slice(0, limit); +} + +function validateSubtasks(value: unknown, malformedReason: RoutingFallbackReason): string[] { + if (!Array.isArray(value) || value.length > COMPOSITIONAL_ROUTER_MAX_SUBTASKS) { + throw new RoutingFailure(malformedReason); + } + const subtasks: string[] = []; + for (const item of value) { + if (typeof item !== "string") throw new RoutingFailure(malformedReason); + const normalized = item.trim(); + if (!normalized || normalized.length > COMPOSITIONAL_ROUTER_MAX_SUBTASK_CHARS) { + throw new RoutingFailure(malformedReason); + } + subtasks.push(normalized); + } + return subtasks; +} + +async function embedSubtasks( + subtasks: readonly string[], + embedder: TextEmbedder, + index: DenseToolIndex, + signal?: AbortSignal, +): Promise { + throwIfAborted(signal); + const embedded = await embedder.embed(subtasks, signal); + throwIfAborted(signal); + const matrix = normalizedMatrix(embedded, subtasks.length); + if (matrix.dimension !== index.dimension) { + throw new TypeError("subtask and catalog embedding dimensions do not match"); + } + return matrix.vectors; +} + +function emptyTimings(): CompositionalRoutingTimings { + return { + initial_decomposition_ms: 0, + catalog_embedding_ms: 0, + initial_retrieval_ms: 0, + refinement_ms: 0, + final_retrieval_ms: 0, + total_ms: 0, + }; +} + +function evidence( + mutable: MutableEvidence, + fallback: RoutingFallbackReason | null, +): CompositionalRoutingEvidence { + return { + initial_subtask_count: mutable.initialSubtaskCount, + refined_subtask_count: mutable.refinedSubtaskCount, + decomposition_passes: mutable.decompositionPasses, + hint_count: mutable.hintNames.length, + hint_tool_names: [...mutable.hintNames], + initial_candidate_counts: mutable.initialCandidates.map((candidates) => candidates.length), + initial_candidate_tool_names: mutable.initialCandidates.map((candidates) => + candidates.map((candidate) => candidate.name), + ), + final_candidate_counts: mutable.finalCandidates.map((candidates) => candidates.length), + final_candidate_tool_names: mutable.finalCandidates.map((candidates) => + candidates.map((candidate) => candidate.name), + ), + selected_tool_count: mutable.selectedNames.length, + selected_tool_names: [...mutable.selectedNames], + fallback, + timings: { ...mutable.timings }, + }; +} + +function uniqueInOrder(values: readonly string[]): string[] { + const seen = new Set(); + return values.filter((value) => { + if (seen.has(value)) return false; + seen.add(value); + return true; + }); +} + +interface RoutingLimits { + topK: number; + hintCount: number; + maxSelectedTools: number; +} + +interface InitialRoutingStage { + initialSubtasks: readonly string[]; + index: DenseToolIndex | null; + mutable: MutableEvidence; + fallback: RoutingFallbackReason | null; +} + +function routingLimits(options: CompositionalRouterOptions): RoutingLimits { + const limits = { + topK: options.topK ?? COMPOSITIONAL_ROUTER_TOP_K, + hintCount: options.hintCount ?? COMPOSITIONAL_ROUTER_HINT_COUNT, + maxSelectedTools: options.maxSelectedTools ?? COMPOSITIONAL_ROUTER_MAX_SELECTED_TOOLS, + }; + validatePositiveInteger(limits.topK, "topK"); + validatePositiveInteger(limits.hintCount, "hintCount"); + validatePositiveInteger(limits.maxSelectedTools, "maxSelectedTools"); + return limits; +} + +function emptyMutableEvidence(): MutableEvidence { + return { + initialSubtaskCount: 0, + refinedSubtaskCount: 0, + decompositionPasses: 0, + hintNames: [], + initialCandidates: [], + finalCandidates: [], + selectedNames: [], + timings: emptyTimings(), + }; +} + +function cloneCandidates(candidates: readonly (readonly RankedTool[])[]): RankedTool[][] { + return candidates.map((ranked) => ranked.map((candidate) => ({ ...candidate }))); +} + +function cloneMutableEvidence(source: MutableEvidence): MutableEvidence { + return { + initialSubtaskCount: source.initialSubtaskCount, + refinedSubtaskCount: source.refinedSubtaskCount, + decompositionPasses: source.decompositionPasses, + hintNames: [...source.hintNames], + initialCandidates: cloneCandidates(source.initialCandidates), + finalCandidates: cloneCandidates(source.finalCandidates), + selectedNames: [...source.selectedNames], + timings: { ...source.timings }, + }; +} + +function fallbackReason(error: unknown): RoutingFallbackReason { + return error instanceof RoutingFailure ? error.reason : "subtask-embedding-failed"; +} + +async function prepareInitialRoutingStage( + query: string, + catalog: readonly ToolRoutingCatalogEntry[], + options: Omit, + limits: RoutingLimits, + clock: () => number, +): Promise { + const mutable = emptyMutableEvidence(); + let initialSubtasks: string[] = []; + let index: DenseToolIndex | null = null; + try { + throwIfAborted(options.signal); + if (typeof query !== "string" || !query.trim()) throw new RoutingFailure("invalid-query"); + validateCatalog(catalog); + + const initialStartedAt = clock(); + let rawInitial: unknown; + mutable.decompositionPasses = 1; + try { + rawInitial = await options.decomposer.decompose({ + pass: "initial", + query, + tool_hints: [], + ...(options.signal ? { signal: options.signal } : {}), + }); + } catch { + throw new RoutingFailure("initial-decomposition-failed"); + } finally { + mutable.timings.initial_decomposition_ms = elapsed(clock, initialStartedAt); + } + initialSubtasks = validateSubtasks(rawInitial, "initial-decomposition-malformed"); + mutable.initialSubtaskCount = initialSubtasks.length; + if (initialSubtasks.length === 0) { + return { initialSubtasks, index, mutable, fallback: null }; + } + + const catalogStartedAt = clock(); + try { + index = await (options.preparedIndex ?? + buildDenseToolIndex(catalog, options.embedder, options.signal)); + const catalogNames = catalog.map((entry) => entry.name); + if ( + index.tools.length !== catalogNames.length || + index.tools.some((tool, position) => tool.name !== catalogNames[position]) + ) { + throw new TypeError("prepared index does not match the routing catalog"); + } + } catch { + throw new RoutingFailure("catalog-embedding-failed"); + } finally { + mutable.timings.catalog_embedding_ms = elapsed(clock, catalogStartedAt); + } + if (index.tools.length === 0) throw new RoutingFailure("no-candidates"); + + const initialRetrievalStartedAt = clock(); + try { + const vectors = await embedSubtasks(initialSubtasks, options.embedder, index, options.signal); + mutable.initialCandidates = vectors.map((vector) => + exactInnerProductTopK( + index as DenseToolIndex, + vector, + Math.max(limits.hintCount, limits.topK), + ), + ); + } catch { + throw new RoutingFailure("subtask-embedding-failed"); + } finally { + mutable.timings.initial_retrieval_ms = elapsed(clock, initialRetrievalStartedAt); + } + return { initialSubtasks, index, mutable, fallback: null }; + } catch (error) { + return { initialSubtasks, index, mutable, fallback: fallbackReason(error) }; + } +} + +async function routeFromInitialStage( + query: string, + catalog: readonly ToolRoutingCatalogEntry[], + options: Omit, + refinementPasses: 0 | 1, + limits: RoutingLimits, + clock: () => number, + totalStartedAt: number, + stage: InitialRoutingStage, +): Promise { + const mutable = cloneMutableEvidence(stage.mutable); + const finish = ( + disposition: CompositionalRoutingResult["disposition"], + fallback: RoutingFallbackReason | null, + ): CompositionalRoutingResult => { + mutable.timings.total_ms = elapsed(clock, totalStartedAt); + return { + disposition, + selected_tool_names: disposition === "routed" ? [...mutable.selectedNames] : [], + evidence: evidence(mutable, fallback), + }; + }; + + if (stage.fallback !== null) return finish("passthrough", stage.fallback); + if (stage.initialSubtasks.length === 0) return finish("routed", null); + const index = stage.index; + if (index === null) return finish("passthrough", "no-candidates"); + + try { + if (refinementPasses === 0) { + mutable.finalCandidates = mutable.initialCandidates.map((candidates) => + candidates.slice(0, limits.topK), + ); + const selectedNames = uniqueInOrder( + mutable.finalCandidates.map((candidates) => candidates[0]?.name).filter(Boolean), + ); + if (selectedNames.length > limits.maxSelectedTools) { + throw new RoutingFailure("selection-limit-exceeded"); + } + mutable.selectedNames = selectedNames; + return finish("routed", null); + } + + const hints = unionToolHints(mutable.initialCandidates, limits.hintCount); + mutable.hintNames = hints.map((candidate) => candidate.name); + if (mutable.hintNames.length === 0) throw new RoutingFailure("no-candidates"); + + const refinementStartedAt = clock(); + let rawRefined: unknown; + mutable.decompositionPasses = 2; + try { + rawRefined = await options.decomposer.decompose({ + pass: "refined", + query, + tool_hints: mutable.hintNames, + ...(options.signal ? { signal: options.signal } : {}), + }); + } catch { + throw new RoutingFailure("refinement-failed"); + } finally { + mutable.timings.refinement_ms = elapsed(clock, refinementStartedAt); + } + const refinedSubtasks = validateSubtasks(rawRefined, "refinement-malformed"); + mutable.refinedSubtaskCount = refinedSubtasks.length; + if (refinedSubtasks.length === 0) { + throw new RoutingFailure("refinement-no-tool-disagreement"); + } + + const finalRetrievalStartedAt = clock(); + try { + const vectors = await embedSubtasks(refinedSubtasks, options.embedder, index, options.signal); + mutable.finalCandidates = vectors.map((vector) => + exactInnerProductTopK(index, vector, limits.topK), + ); + } catch { + throw new RoutingFailure("subtask-embedding-failed"); + } finally { + mutable.timings.final_retrieval_ms = elapsed(clock, finalRetrievalStartedAt); + } + if (mutable.finalCandidates.some((candidates) => candidates.length === 0)) { + throw new RoutingFailure("no-candidates"); + } + + const selectedNames = uniqueInOrder( + mutable.finalCandidates.map((candidates) => candidates[0]?.name).filter(Boolean), + ); + if (selectedNames.length > limits.maxSelectedTools) { + throw new RoutingFailure("selection-limit-exceeded"); + } + mutable.selectedNames = selectedNames; + return finish("routed", null); + } catch (error) { + return finish("passthrough", fallbackReason(error)); + } +} + +/** + * Run an initial decomposition and, by default, one tool-informed refinement. + * Set `refinementPasses` to zero to score the initial route. Any malformed model + * output or unavailable retrieval dependency returns `passthrough`, so a caller + * can retain the complete original tool catalog rather than losing a capability. + * An empty initial decomposition is treated as an intentional no-tool route. + */ +export async function routeCompositionalTools( + query: string, + catalog: readonly ToolRoutingCatalogEntry[], + options: CompositionalRouterOptions, +): Promise { + const limits = routingLimits(options); + const refinementPasses = options.refinementPasses ?? 1; + if (refinementPasses !== 0 && refinementPasses !== 1) { + throw new TypeError("refinementPasses must be zero or one"); + } + const clock = options.clock ?? (() => performance.now()); + const totalStartedAt = clock(); + const sharedOptions = { ...options, refinementPasses: undefined }; + const stage = await prepareInitialRoutingStage(query, catalog, sharedOptions, limits, clock); + return routeFromInitialStage( + query, + catalog, + sharedOptions, + refinementPasses, + limits, + clock, + totalStartedAt, + stage, + ); +} + +/** + * Produce a fair initial/refined pair from one initial decomposition and one + * shared catalog index and initial retrieval stage. + */ +export async function routeCompositionalToolsPaired( + query: string, + catalog: readonly ToolRoutingCatalogEntry[], + options: Omit, +): Promise { + const limits = routingLimits(options); + const clock = options.clock ?? (() => performance.now()); + const totalStartedAt = clock(); + const stage = await prepareInitialRoutingStage(query, catalog, options, limits, clock); + const initial = await routeFromInitialStage( + query, + catalog, + options, + 0, + limits, + clock, + totalStartedAt, + stage, + ); + const refined = await routeFromInitialStage( + query, + catalog, + options, + 1, + limits, + clock, + totalStartedAt, + stage, + ); + if (initial.evidence.initial_subtask_count !== refined.evidence.initial_subtask_count) { + throw new Error("paired routes did not share the initial decomposition"); + } + return { + initial, + refined, + shared_initial_decomposition: true, + shared_initial_subtask_count: initial.evidence.initial_subtask_count, + }; +} + +/** + * Resolve selected names back to the caller-owned definitions. Routed results + * preserve selected order; passthrough results preserve the full catalog order. + */ +export function selectRoutedToolDefinitions( + catalog: readonly ToolRoutingCatalogEntry[], + result: CompositionalRoutingResult, +): Definition[] { + if (result.disposition === "passthrough") return catalog.map((entry) => entry.definition); + const byName = new Map(catalog.map((entry) => [entry.name, entry.definition])); + return result.selected_tool_names.flatMap((name) => { + const definition = byName.get(name); + return definition === undefined ? [] : [definition]; + }); +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-acceptance.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-acceptance.ts new file mode 100644 index 0000000000..4ef96345b6 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-acceptance.ts @@ -0,0 +1,649 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { OpenAIFunctionTool } from "./catalog"; +import type { RoutingFallbackReason } from "./compositional-tool-router"; + +export type CompositionalRoutingVariant = "initial" | "refined"; + +export interface CompositionalRoutingAcceptanceTool { + name: string; + description: string; + definition: OpenAIFunctionTool; +} + +export interface CompositionalRoutingExpectedStep { + capability: string; + /** A lower-case phrase that makes this capability observable in the prompt. */ + prompt_cue: string; + tool_name: string; +} + +export interface CompositionalRoutingAcceptanceCase { + id: string; + prompt: string; + expected_steps: readonly CompositionalRoutingExpectedStep[]; +} + +export interface CompositionalRoutingStepPrediction { + subtask: string; + /** Best candidate first. The evaluator reads positions 1 and 1-10. */ + ranked_tool_names: readonly string[]; +} + +export interface CompositionalRoutingCasePrediction { + case_id: string; + disposition: "routed" | "passthrough"; + fallback: RoutingFallbackReason | null; + steps: readonly CompositionalRoutingStepPrediction[]; + /** Logical router selection; passthrough forwarding is derived from the catalog. */ + selected_tool_names: readonly string[]; +} + +export interface CompositionalRoutingVariantInput { + variant: CompositionalRoutingVariant; + cases: readonly CompositionalRoutingCasePrediction[]; +} + +export interface CompositionalRoutingCaseResult { + case_id: string; + disposition: "routed" | "passthrough"; + fallback: RoutingFallbackReason | null; + routing_succeeded: boolean; + expected_step_count: number; + predicted_step_count: number; + decomposition_exact: boolean; + decomposition_within_one: boolean; + /** One-based rank for each expected step; null means the target was absent. */ + expected_tool_ranks: readonly (number | null)[]; + chain_exact_selection: boolean; + selected_tool_count: number; + selection_count_exact: boolean; + forwarded_tool_names: readonly string[]; + forwarded_tool_count: number; + chain_exact_forwarding: boolean; + forwarding_count_exact: boolean; + /** Null for cases that require tools. */ + no_tool_exact: boolean | null; +} + +export interface CompositionalRoutingMetrics { + variant: CompositionalRoutingVariant; + case_count: number; + route_failure_case_count: number; + routing_success_rate: number; + expected_step_count: number; + predicted_step_count: number; + decomposition_exact_rate: number; + decomposition_within_one_rate: number; + exact_tool_recall_at_1: number; + exact_tool_recall_at_10: number; + chain_exact_selection_rate: number; + selection_count_exact_rate: number; + chain_exact_forwarding_rate: number; + forwarding_count_exact_rate: number; + selected_tool_count: number; + mean_selected_tool_count: number; + max_selected_tool_count: number; + forwarded_tool_count: number; + mean_forwarded_tool_count: number; + max_forwarded_tool_count: number; + no_tool_case_count: number; + no_tool_exact_rate: number; + cases: readonly CompositionalRoutingCaseResult[]; +} + +export interface CompositionalRoutingGateThresholds { + min_routing_success_rate: number; + min_decomposition_exact_rate: number; + min_decomposition_within_one_rate: number; + min_exact_tool_recall_at_1: number; + min_exact_tool_recall_at_10: number; + min_chain_exact_selection_rate: number; + min_selection_count_exact_rate: number; + min_chain_exact_forwarding_rate: number; + min_forwarding_count_exact_rate: number; + min_no_tool_exact_rate: number; + max_selected_tool_count: number; + max_forwarded_tool_count: number; +} + +export interface CompositionalRoutingGateResult { + passed: boolean; + thresholds: CompositionalRoutingGateThresholds; + reasons: readonly string[]; +} + +export interface CompositionalRoutingComparison { + initial: CompositionalRoutingMetrics; + refined: CompositionalRoutingMetrics; + refined_minus_initial: { + decomposition_exact_rate: number; + decomposition_within_one_rate: number; + exact_tool_recall_at_1: number; + exact_tool_recall_at_10: number; + chain_exact_selection_rate: number; + chain_exact_forwarding_rate: number; + routing_success_rate: number; + }; + refined_not_worse: boolean; + refined_gate: CompositionalRoutingGateResult; + passed: boolean; + reasons: readonly string[]; +} + +export const DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS: CompositionalRoutingGateThresholds = { + min_routing_success_rate: 1, + min_decomposition_exact_rate: 1, + min_decomposition_within_one_rate: 1, + min_exact_tool_recall_at_1: 1, + min_exact_tool_recall_at_10: 1, + min_chain_exact_selection_rate: 1, + min_selection_count_exact_rate: 1, + min_chain_exact_forwarding_rate: 1, + min_forwarding_count_exact_rate: 1, + min_no_tool_exact_rate: 1, + max_selected_tool_count: 5, + max_forwarded_tool_count: 5, +}; + +function routeTool(name: string, description: string): CompositionalRoutingAcceptanceTool { + return { + name, + description, + definition: { + type: "function", + function: { + name, + description, + parameters: { + type: "object", + description: "Route-only acceptance fixture input.", + properties: { + input: { + type: "string", + description: "Deterministic input for the selected fixture capability.", + minLength: 1, + maxLength: 160, + }, + }, + required: ["input"], + additionalProperties: false, + }, + }, + }, + }; +} + +/** + * A small route-only catalog with deliberately distinct capability metadata. + * It complements the scaling catalog, whose repeated category/operation pairs + * are useful for schema-volume tests but cannot identify one exact semantic + * target without relying on numeric suffixes. + */ +export const COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS: readonly CompositionalRoutingAcceptanceTool[] = + [ + routeTool( + "route_fetch_https_bytes", + "Download raw response bytes from one HTTPS address without interpreting the payload.", + ), + routeTool( + "route_read_local_file", + "Read bytes from a file already present on the local filesystem.", + ), + routeTool( + "route_decompress_gzip", + "Expand a gzip-compressed byte stream into its original uncompressed bytes.", + ), + routeTool("route_decompress_zip", "Extract named members from a ZIP archive container."), + routeTool( + "route_parse_csv_rows", + "Parse comma-separated text with a header row into structured records.", + ), + routeTool( + "route_parse_jsonl_records", + "Parse newline-delimited JSON text into one structured record per line.", + ), + routeTool( + "route_validate_json_schema", + "Check structured records against a supplied JSON Schema contract.", + ), + routeTool( + "route_filter_tabular_rows", + "Keep only table rows that satisfy a field-level predicate.", + ), + routeTool( + "route_aggregate_numeric_column", + "Compute a requested numeric aggregate such as an average over one table column.", + ), + routeTool("route_render_line_chart", "Render ordered numeric observations as a line chart."), + routeTool("route_render_bar_chart", "Render categorical numeric values as vertical bars."), + routeTool( + "route_lookup_directory_contact", + "Find a person's messaging destination in an organization directory.", + ), + routeTool( + "route_draft_email_message", + "Create an email draft with recipients, subject, and body without sending it.", + ), + routeTool("route_send_email_message", "Deliver an existing email draft to its recipients."), + routeTool("route_post_team_channel", "Post a text update to a team chat channel destination."), + routeTool("route_store_object_blob", "Write supplied bytes to an object-storage key."), + routeTool( + "route_query_sql_readonly", + "Run a read-only SQL query and return tabular result rows.", + ), + routeTool("route_summarize_text", "Condense a text document into a short prose summary."), + routeTool("route_transcribe_audio", "Convert spoken audio into a text transcript."), + routeTool("route_translate_text", "Translate supplied text from one language to another."), + ]; + +function step( + toolName: string, + capability: string, + promptCue: string, +): CompositionalRoutingExpectedStep { + return { tool_name: toolName, capability, prompt_cue: promptCue }; +} + +/** Eight unambiguous cases spanning zero through five implicit atomic capabilities. */ +export const COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES: readonly CompositionalRoutingAcceptanceCase[] = + [ + { + id: "route-no-tool-01", + prompt: "Use no external capability and reply with the literal phrase ROUTE-CONTROL-ALPHA.", + expected_steps: [], + }, + { + id: "route-single-csv-01", + prompt: "Turn comma-separated text with a header row into structured records.", + expected_steps: [step("route_parse_csv_rows", "parse CSV records", "comma-separated")], + }, + { + id: "route-single-bars-01", + prompt: "Show the categorical totals as vertical bars, rather than a connected line.", + expected_steps: [ + step( + "route_render_bar_chart", + "render categorical values as vertical bars", + "vertical bars", + ), + ], + }, + { + id: "route-chain-two-channel-01", + prompt: + "Find Morgan's team-chat destination in the organization directory, then post the status update to that channel.", + expected_steps: [ + step("route_lookup_directory_contact", "look up a directory contact", "directory"), + step("route_post_team_channel", "post a team-channel message", "post"), + ], + }, + { + id: "route-chain-two-email-01", + prompt: "Prepare an email draft for the reviewers, then deliver that draft to them.", + expected_steps: [ + step("route_draft_email_message", "draft an email", "draft"), + step("route_send_email_message", "send an email draft", "deliver"), + ], + }, + { + id: "route-chain-three-analysis-01", + prompt: + "Run a read-only SQL query for monthly latency, calculate the average value, and plot the ordered results as a line chart.", + expected_steps: [ + step("route_query_sql_readonly", "query SQL data", "sql"), + step("route_aggregate_numeric_column", "aggregate a numeric column", "average"), + step("route_render_line_chart", "render a line chart", "line chart"), + ], + }, + { + id: "route-chain-four-jsonl-01", + prompt: + "Download the bytes at the HTTPS address, expand the gzip stream, parse its newline-delimited JSON records, and check them against the supplied JSON Schema.", + expected_steps: [ + step("route_fetch_https_bytes", "fetch HTTPS bytes", "https"), + step("route_decompress_gzip", "decompress gzip bytes", "gzip"), + step("route_parse_jsonl_records", "parse JSONL records", "newline-delimited json"), + step("route_validate_json_schema", "validate JSON records", "json schema"), + ], + }, + { + id: "route-chain-five-report-01", + prompt: + "Fetch the CSV file from its HTTPS address, parse the comma-separated rows, keep rows whose state is active, average the cost column, and present the result as vertical bars.", + expected_steps: [ + step("route_fetch_https_bytes", "fetch HTTPS bytes", "https"), + step("route_parse_csv_rows", "parse CSV records", "comma-separated"), + step("route_filter_tabular_rows", "filter table rows", "keep rows"), + step("route_aggregate_numeric_column", "aggregate a numeric column", "average"), + step( + "route_render_bar_chart", + "render categorical values as vertical bars", + "vertical bars", + ), + ], + }, + ]; + +function assertRate(value: number, label: string): void { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new RangeError(`${label} must be between zero and one`); + } +} + +function assertThresholds(thresholds: CompositionalRoutingGateThresholds): void { + for (const [label, value] of Object.entries(thresholds)) { + if (label === "max_selected_tool_count" || label === "max_forwarded_tool_count") continue; + assertRate(value, label); + } + for (const [label, value] of [ + ["max_selected_tool_count", thresholds.max_selected_tool_count], + ["max_forwarded_tool_count", thresholds.max_forwarded_tool_count], + ] as const) { + if (!Number.isSafeInteger(value)) throw new TypeError(`${label} must be a safe integer`); + if (value < 0) throw new RangeError(`${label} must not be negative`); + } +} + +function assertUnique(values: readonly string[], label: string): void { + if (new Set(values).size !== values.length) throw new Error(`${label} contains duplicates`); +} + +function assertFixture( + tools: readonly CompositionalRoutingAcceptanceTool[], + cases: readonly CompositionalRoutingAcceptanceCase[], +): Set { + if (tools.length === 0) throw new Error("route acceptance catalog must not be empty"); + if (cases.length === 0) throw new Error("route acceptance cases must not be empty"); + const toolNames = tools.map((tool) => tool.name); + assertUnique(toolNames, "route acceptance tool names"); + const knownTools = new Set(toolNames); + const caseIds = cases.map((fixture) => fixture.id); + assertUnique(caseIds, "route acceptance case ids"); + for (const fixture of cases) { + if (!fixture.prompt.trim()) throw new Error(`${fixture.id} has an empty prompt`); + const expectedNames = fixture.expected_steps.map((expected) => expected.tool_name); + assertUnique(expectedNames, `${fixture.id} expected tool names`); + for (const expected of fixture.expected_steps) { + if (!knownTools.has(expected.tool_name)) { + throw new Error(`${fixture.id} references unknown tool ${expected.tool_name}`); + } + if (!expected.capability.trim() || !expected.prompt_cue.trim()) { + throw new Error(`${fixture.id} has empty expected step metadata`); + } + if (!fixture.prompt.toLocaleLowerCase("en-US").includes(expected.prompt_cue)) { + throw new Error(`${fixture.id} does not expose prompt cue ${expected.prompt_cue}`); + } + } + } + return knownTools; +} + +function assertPrediction( + prediction: CompositionalRoutingCasePrediction, + knownTools: ReadonlySet, +): void { + if (prediction.disposition !== "routed" && prediction.disposition !== "passthrough") { + throw new TypeError(`${prediction.case_id} has an invalid disposition`); + } + const routingSucceeded = prediction.disposition === "routed"; + if (routingSucceeded !== (prediction.fallback === null)) { + throw new Error(`${prediction.case_id} has inconsistent disposition and fallback`); + } + assertUnique(prediction.selected_tool_names, `${prediction.case_id} selected tools`); + for (const name of prediction.selected_tool_names) { + if (!knownTools.has(name)) { + throw new Error(`${prediction.case_id} selected unknown tool ${name}`); + } + } + for (const [index, predicted] of prediction.steps.entries()) { + if (!predicted.subtask.trim()) { + throw new Error(`${prediction.case_id} step ${index + 1} has an empty subtask`); + } + assertUnique(predicted.ranked_tool_names, `${prediction.case_id} step ${index + 1} ranking`); + for (const name of predicted.ranked_tool_names) { + if (!knownTools.has(name)) { + throw new Error(`${prediction.case_id} ranked unknown tool ${name}`); + } + } + } +} + +function rate(numerator: number, denominator: number): number { + return denominator === 0 ? 1 : numerator / denominator; +} + +function sequencesEqual(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +/** Evaluate one router variant without invoking a model or embedding implementation. */ +export function evaluateCompositionalRoutingVariant( + input: CompositionalRoutingVariantInput, + options: { + tools?: readonly CompositionalRoutingAcceptanceTool[]; + cases?: readonly CompositionalRoutingAcceptanceCase[]; + } = {}, +): CompositionalRoutingMetrics { + const tools = options.tools ?? COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS; + const fixtures = options.cases ?? COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES; + const knownTools = assertFixture(tools, fixtures); + const catalogToolNames = tools.map((tool) => tool.name); + if (input.cases.length !== fixtures.length) { + throw new Error(`${input.variant} predictions must contain exactly ${fixtures.length} cases`); + } + const predictions = new Map(); + for (const prediction of input.cases) { + if (predictions.has(prediction.case_id)) { + throw new Error(`${input.variant} contains duplicate case ${prediction.case_id}`); + } + assertPrediction(prediction, knownTools); + predictions.set(prediction.case_id, prediction); + } + + const results: CompositionalRoutingCaseResult[] = []; + let expectedStepCount = 0; + let predictedStepCount = 0; + let routingSuccesses = 0; + let routeFailureCases = 0; + let exactDecomposition = 0; + let withinOneDecomposition = 0; + let recallAtOne = 0; + let recallAtTen = 0; + let exactChains = 0; + let exactSelectionCounts = 0; + let exactForwardingChains = 0; + let exactForwardingCounts = 0; + let selectedToolCount = 0; + let maxSelectedToolCount = 0; + let forwardedToolCount = 0; + let maxForwardedToolCount = 0; + let noToolCaseCount = 0; + let exactNoToolCases = 0; + + for (const fixture of fixtures) { + const prediction = predictions.get(fixture.id); + if (!prediction) throw new Error(`${input.variant} is missing case ${fixture.id}`); + const expectedNames = fixture.expected_steps.map((expected) => expected.tool_name); + const expectedCount = expectedNames.length; + const predictedCount = prediction.steps.length; + const routingSucceeded = prediction.disposition === "routed"; + const forwardedToolNames = routingSucceeded + ? [...prediction.selected_tool_names] + : [...catalogToolNames]; + const decompositionExact = predictedCount === expectedCount; + const decompositionWithinOne = Math.abs(predictedCount - expectedCount) <= 1; + const expectedToolRanks = expectedNames.map((expectedName, index) => { + const ranked = prediction.steps[index]?.ranked_tool_names ?? []; + const rank = ranked.indexOf(expectedName); + if (rank === 0) recallAtOne += 1; + if (rank >= 0 && rank < 10) recallAtTen += 1; + return rank < 0 ? null : rank + 1; + }); + const chainExactSelection = sequencesEqual(prediction.selected_tool_names, expectedNames); + const selectionCountExact = prediction.selected_tool_names.length === expectedCount; + const chainExactForwarding = + routingSucceeded && sequencesEqual(forwardedToolNames, expectedNames); + const forwardingCountExact = routingSucceeded && forwardedToolNames.length === expectedCount; + const noToolExact = + expectedCount === 0 + ? routingSucceeded && predictedCount === 0 && forwardedToolNames.length === 0 + : null; + + expectedStepCount += expectedCount; + predictedStepCount += predictedCount; + routingSuccesses += Number(routingSucceeded); + routeFailureCases += Number(!routingSucceeded); + exactDecomposition += Number(decompositionExact); + withinOneDecomposition += Number(decompositionWithinOne); + exactChains += Number(chainExactSelection); + exactSelectionCounts += Number(selectionCountExact); + exactForwardingChains += Number(chainExactForwarding); + exactForwardingCounts += Number(forwardingCountExact); + selectedToolCount += prediction.selected_tool_names.length; + maxSelectedToolCount = Math.max(maxSelectedToolCount, prediction.selected_tool_names.length); + forwardedToolCount += forwardedToolNames.length; + maxForwardedToolCount = Math.max(maxForwardedToolCount, forwardedToolNames.length); + if (noToolExact !== null) { + noToolCaseCount += 1; + exactNoToolCases += Number(noToolExact); + } + results.push({ + case_id: fixture.id, + disposition: prediction.disposition, + fallback: prediction.fallback, + routing_succeeded: routingSucceeded, + expected_step_count: expectedCount, + predicted_step_count: predictedCount, + decomposition_exact: decompositionExact, + decomposition_within_one: decompositionWithinOne, + expected_tool_ranks: expectedToolRanks, + chain_exact_selection: chainExactSelection, + selected_tool_count: prediction.selected_tool_names.length, + selection_count_exact: selectionCountExact, + forwarded_tool_names: forwardedToolNames, + forwarded_tool_count: forwardedToolNames.length, + chain_exact_forwarding: chainExactForwarding, + forwarding_count_exact: forwardingCountExact, + no_tool_exact: noToolExact, + }); + } + + return { + variant: input.variant, + case_count: fixtures.length, + route_failure_case_count: routeFailureCases, + routing_success_rate: rate(routingSuccesses, fixtures.length), + expected_step_count: expectedStepCount, + predicted_step_count: predictedStepCount, + decomposition_exact_rate: rate(exactDecomposition, fixtures.length), + decomposition_within_one_rate: rate(withinOneDecomposition, fixtures.length), + exact_tool_recall_at_1: rate(recallAtOne, expectedStepCount), + exact_tool_recall_at_10: rate(recallAtTen, expectedStepCount), + chain_exact_selection_rate: rate(exactChains, fixtures.length), + selection_count_exact_rate: rate(exactSelectionCounts, fixtures.length), + chain_exact_forwarding_rate: rate(exactForwardingChains, fixtures.length), + forwarding_count_exact_rate: rate(exactForwardingCounts, fixtures.length), + selected_tool_count: selectedToolCount, + mean_selected_tool_count: selectedToolCount / fixtures.length, + max_selected_tool_count: maxSelectedToolCount, + forwarded_tool_count: forwardedToolCount, + mean_forwarded_tool_count: forwardedToolCount / fixtures.length, + max_forwarded_tool_count: maxForwardedToolCount, + no_tool_case_count: noToolCaseCount, + no_tool_exact_rate: rate(exactNoToolCases, noToolCaseCount), + cases: results, + }; +} + +/** Apply explicit, deterministic quality and disclosure-size gates. */ +export function evaluateCompositionalRoutingGates( + metrics: CompositionalRoutingMetrics, + thresholds: CompositionalRoutingGateThresholds = DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS, +): CompositionalRoutingGateResult { + assertThresholds(thresholds); + const reasons: string[] = []; + if (metrics.route_failure_case_count > 0) { + reasons.push(`route_failure_case_count ${metrics.route_failure_case_count} must be zero`); + } + const minimums: ReadonlyArray< + [keyof CompositionalRoutingMetrics, keyof CompositionalRoutingGateThresholds] + > = [ + ["routing_success_rate", "min_routing_success_rate"], + ["decomposition_exact_rate", "min_decomposition_exact_rate"], + ["decomposition_within_one_rate", "min_decomposition_within_one_rate"], + ["exact_tool_recall_at_1", "min_exact_tool_recall_at_1"], + ["exact_tool_recall_at_10", "min_exact_tool_recall_at_10"], + ["chain_exact_selection_rate", "min_chain_exact_selection_rate"], + ["selection_count_exact_rate", "min_selection_count_exact_rate"], + ["chain_exact_forwarding_rate", "min_chain_exact_forwarding_rate"], + ["forwarding_count_exact_rate", "min_forwarding_count_exact_rate"], + ["no_tool_exact_rate", "min_no_tool_exact_rate"], + ]; + for (const [metricName, thresholdName] of minimums) { + const value = metrics[metricName]; + const threshold = thresholds[thresholdName]; + if (typeof value !== "number" || value < threshold) { + reasons.push(`${String(metricName)} ${String(value)} is below ${String(threshold)}`); + } + } + if (metrics.max_selected_tool_count > thresholds.max_selected_tool_count) { + reasons.push( + `max_selected_tool_count ${metrics.max_selected_tool_count} exceeds ${thresholds.max_selected_tool_count}`, + ); + } + if (metrics.max_forwarded_tool_count > thresholds.max_forwarded_tool_count) { + reasons.push( + `max_forwarded_tool_count ${metrics.max_forwarded_tool_count} exceeds ${thresholds.max_forwarded_tool_count}`, + ); + } + return { passed: reasons.length === 0, thresholds: { ...thresholds }, reasons }; +} + +/** Compare initial and one-pass refined routing while gating refined independently. */ +export function compareCompositionalRoutingVariants( + initialInput: CompositionalRoutingVariantInput, + refinedInput: CompositionalRoutingVariantInput, + options: { + tools?: readonly CompositionalRoutingAcceptanceTool[]; + cases?: readonly CompositionalRoutingAcceptanceCase[]; + thresholds?: CompositionalRoutingGateThresholds; + } = {}, +): CompositionalRoutingComparison { + if (initialInput.variant !== "initial") { + throw new Error("the first route comparison input must be initial"); + } + if (refinedInput.variant !== "refined") { + throw new Error("the second route comparison input must be refined"); + } + const evaluateOptions = { tools: options.tools, cases: options.cases }; + const initial = evaluateCompositionalRoutingVariant(initialInput, evaluateOptions); + const refined = evaluateCompositionalRoutingVariant(refinedInput, evaluateOptions); + const refinedGate = evaluateCompositionalRoutingGates(refined, options.thresholds); + const deltas = { + decomposition_exact_rate: refined.decomposition_exact_rate - initial.decomposition_exact_rate, + decomposition_within_one_rate: + refined.decomposition_within_one_rate - initial.decomposition_within_one_rate, + exact_tool_recall_at_1: refined.exact_tool_recall_at_1 - initial.exact_tool_recall_at_1, + exact_tool_recall_at_10: refined.exact_tool_recall_at_10 - initial.exact_tool_recall_at_10, + chain_exact_selection_rate: + refined.chain_exact_selection_rate - initial.chain_exact_selection_rate, + chain_exact_forwarding_rate: + refined.chain_exact_forwarding_rate - initial.chain_exact_forwarding_rate, + routing_success_rate: refined.routing_success_rate - initial.routing_success_rate, + }; + const regressions = Object.entries(deltas) + .filter(([, value]) => value < 0) + .map(([name, value]) => `${name} regressed by ${Math.abs(value)}`); + const reasons = [...refinedGate.reasons, ...regressions]; + return { + initial, + refined, + refined_minus_initial: deltas, + refined_not_worse: regressions.length === 0, + refined_gate: refinedGate, + passed: reasons.length === 0, + reasons, + }; +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts new file mode 100644 index 0000000000..bc0a4ccad5 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts @@ -0,0 +1,375 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { performance } from "node:perf_hooks"; + +import type { DecompositionPass, TaskDecomposer, TextEmbedder } from "./compositional-tool-router"; + +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_MAX_OUTPUT_TOKENS = 256; +const DEFAULT_EMBEDDING_BATCH_SIZE = 128; +const DEFAULT_HASH_DIMENSIONS = 1_024; +const MAX_TEXT_CHARS = 32_768; +const MAX_RESPONSE_BYTES = 8 * 1024 * 1024; + +interface JsonRecord { + [key: string]: unknown; +} + +export interface ModelUsageEvent { + operation: "decomposition" | "embedding"; + pass?: DecompositionPass; + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + duration_ms: number; +} + +interface OpenAIAdapterOptions { + baseUrl: string; + model: string; + apiKey?: string; + /** Required for non-loopback HTTPS endpoints because prompts leave the host. */ + allowRemote?: boolean; + timeoutMs?: number; + onUsage?: (event: ModelUsageEvent) => void; +} + +export interface OpenAIChatDecomposerOptions extends OpenAIAdapterOptions { + maxOutputTokens?: number; +} + +export interface OpenAITextEmbedderOptions extends OpenAIAdapterOptions { + batchSize?: number; +} + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function positiveInteger(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new TypeError(`${label} must be a positive safe integer`); + } + return value; +} + +function endpoint( + baseUrl: string, + leaf: "chat/completions" | "embeddings", + allowRemote: boolean, +): string { + const value = new URL(baseUrl); + if (value.protocol !== "http:" && value.protocol !== "https:") { + throw new TypeError("model adapter baseUrl must use HTTP or HTTPS"); + } + if (value.username || value.password || value.search || value.hash) { + throw new TypeError("model adapter baseUrl must not contain credentials, query, or fragment"); + } + const hostname = value.hostname.replace(/^\[|\]$/gu, "").toLowerCase(); + const loopback = + hostname === "localhost" || hostname === "::1" || /^127(?:\.\d{1,3}){3}$/u.test(hostname); + if (!loopback && value.protocol !== "https:") { + throw new TypeError("remote model adapter endpoints must use HTTPS"); + } + if (!loopback && !allowRemote) { + throw new TypeError("remote model adapter endpoints require allowRemote"); + } + const path = value.pathname.replace(/\/+$/u, ""); + value.pathname = `${path.endsWith("/v1") ? path : `${path}/v1`}/${leaf}`.replace(/\/{2,}/gu, "/"); + return value.toString(); +} + +async function readBoundedJson(response: Response): Promise { + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > MAX_RESPONSE_BYTES) { + throw new Error("model adapter response is too large"); + } + if (!response.body) return JSON.parse(await response.text()) as unknown; + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > MAX_RESPONSE_BYTES) { + await reader.cancel(); + throw new Error("model adapter response is too large"); + } + chunks.push(value); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return JSON.parse(new TextDecoder().decode(bytes)) as unknown; +} + +function requestHeaders(apiKey: string | undefined): Record { + return { + "content-type": "application/json", + ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}), + }; +} + +function boundedSignal(timeoutMs: number, parent?: AbortSignal): AbortSignal { + const timeout = AbortSignal.timeout(timeoutMs); + return parent ? AbortSignal.any([parent, timeout]) : timeout; +} + +function usageFrom( + value: unknown, +): Pick { + if (!isRecord(value)) return {}; + const result: Pick = {}; + for (const key of ["prompt_tokens", "completion_tokens", "total_tokens"] as const) { + const item = value[key]; + if (typeof item === "number" && Number.isSafeInteger(item) && item >= 0) result[key] = item; + } + return result; +} + +function reportUsage(callback: OpenAIAdapterOptions["onUsage"], event: ModelUsageEvent): void { + try { + callback?.(event); + } catch { + // Measurement callbacks must not change routing behavior. + } +} + +function boundedText(value: string, label: string): string { + const normalized = value.trim(); + if (!normalized || normalized.length > MAX_TEXT_CHARS) { + throw new TypeError(`${label} must contain 1-${MAX_TEXT_CHARS} characters`); + } + return normalized; +} + +function decompositionMessages( + query: string, + pass: DecompositionPass, + hints: readonly string[], +): Array<{ role: "system" | "user"; content: string }> { + const system = [ + "Return only a JSON array of strings.", + "Split the request into a short ordered list of concrete actions.", + "Each action must be solvable with one external capability.", + "Return an empty array when the request needs no external capability.", + ].join(" "); + const user = + pass === "initial" + ? `Identify the atomic actions in this request:\n${query}` + : [ + "Reconsider the action boundaries using these capability names as vocabulary hints:", + hints.join(", "), + "Request:", + query, + ].join("\n"); + return [ + { role: "system", content: system }, + { role: "user", content: user }, + ]; +} + +function jsonPayloadText(value: string): string { + const trimmed = value.trim(); + if (!trimmed.startsWith("```")) return trimmed; + const firstNewline = trimmed.indexOf("\n"); + const lastFence = trimmed.lastIndexOf("```"); + if (firstNewline < 0 || lastFence <= firstNewline) return trimmed; + return trimmed.slice(firstNewline + 1, lastFence).trim(); +} + +function parseDecompositionResponse(value: unknown): unknown { + if (!isRecord(value) || !Array.isArray(value.choices)) { + throw new Error("decomposition response has an unsupported shape"); + } + const first = value.choices[0]; + const content = isRecord(first) && isRecord(first.message) ? first.message.content : null; + if (typeof content !== "string") { + throw new Error("decomposition response has no text content"); + } + let parsed: unknown; + try { + parsed = JSON.parse(jsonPayloadText(content)); + } catch { + throw new Error("decomposition response contains invalid JSON"); + } + return isRecord(parsed) && Array.isArray(parsed.subtasks) ? parsed.subtasks : parsed; +} + +/** Build the two-pass decomposer using any OpenAI-compatible chat endpoint. */ +export function createOpenAIChatTaskDecomposer( + options: OpenAIChatDecomposerOptions, +): TaskDecomposer { + const target = endpoint(options.baseUrl, "chat/completions", options.allowRemote === true); + const model = boundedText(options.model, "decomposition model"); + const timeoutMs = positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS, "timeoutMs"); + const maxOutputTokens = positiveInteger( + options.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS, + "maxOutputTokens", + ); + return { + decompose: async (request) => { + const query = boundedText(request.query, "decomposition query"); + const startedAt = performance.now(); + let body: unknown; + try { + const response = await fetch(target, { + method: "POST", + headers: requestHeaders(options.apiKey), + body: JSON.stringify({ + model, + messages: decompositionMessages(query, request.pass, request.tool_hints), + temperature: 0, + max_tokens: maxOutputTokens, + stream: false, + }), + signal: boundedSignal(timeoutMs, request.signal), + }); + if (!response.ok) throw new Error("decomposition request failed"); + body = await readBoundedJson(response); + } catch { + throw new Error("decomposition request failed"); + } finally { + // Never include prompts, endpoints, model IDs, keys, or response text. + const usage = isRecord(body) ? usageFrom(body.usage) : {}; + reportUsage(options.onUsage, { + operation: "decomposition", + pass: request.pass, + ...usage, + duration_ms: performance.now() - startedAt, + }); + } + return parseDecompositionResponse(body); + }, + }; +} + +function parseEmbeddingResponse(value: unknown, expected: number): number[][] { + if (!isRecord(value) || !Array.isArray(value.data) || value.data.length !== expected) { + throw new Error("embedding response has an unsupported shape"); + } + const ordered = [...value.data].sort((left, right) => { + const leftIndex = isRecord(left) && typeof left.index === "number" ? left.index : -1; + const rightIndex = isRecord(right) && typeof right.index === "number" ? right.index : -1; + return leftIndex - rightIndex; + }); + return ordered.map((item, expectedIndex) => { + if (!isRecord(item) || item.index !== expectedIndex || !Array.isArray(item.embedding)) { + throw new Error("embedding response has an unsupported shape"); + } + const vector = item.embedding; + if ( + vector.length === 0 || + vector.some((number) => typeof number !== "number" || !Number.isFinite(number)) + ) { + throw new Error("embedding response has an invalid vector"); + } + return vector as number[]; + }); +} + +/** Build a dense embedder using an OpenAI-compatible embeddings endpoint. */ +export function createOpenAITextEmbedder(options: OpenAITextEmbedderOptions): TextEmbedder { + const target = endpoint(options.baseUrl, "embeddings", options.allowRemote === true); + const model = boundedText(options.model, "embedding model"); + const timeoutMs = positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS, "timeoutMs"); + const batchSize = positiveInteger(options.batchSize ?? DEFAULT_EMBEDDING_BATCH_SIZE, "batchSize"); + return { + embed: async (texts, signal) => { + const output: number[][] = []; + for (let start = 0; start < texts.length; start += batchSize) { + const input = texts + .slice(start, start + batchSize) + .map((text) => boundedText(text, "embedding input")); + const startedAt = performance.now(); + let body: unknown; + try { + const response = await fetch(target, { + method: "POST", + headers: requestHeaders(options.apiKey), + body: JSON.stringify({ model, input }), + signal: boundedSignal(timeoutMs, signal), + }); + if (!response.ok) throw new Error("embedding request failed"); + body = await readBoundedJson(response); + output.push(...parseEmbeddingResponse(body, input.length)); + } catch { + throw new Error("embedding request failed"); + } finally { + const usage = isRecord(body) ? usageFrom(body.usage) : {}; + reportUsage(options.onUsage, { + operation: "embedding", + ...usage, + duration_ms: performance.now() - startedAt, + }); + } + } + return output; + }, + }; +} + +function fnv1a(text: string): number { + let hash = 0x81_1c_9d_c5; + for (let index = 0; index < text.length; index += 1) { + hash ^= text.charCodeAt(index); + hash = Math.imul(hash, 0x01_00_01_93); + } + return hash >>> 0; +} + +function lexicalFeatures(text: string): string[] { + const words = text.toLowerCase().match(/[a-z0-9]+/gu) ?? []; + const features = [...words]; + for (let index = 0; index + 1 < words.length; index += 1) { + features.push(`${words[index]}_${words[index + 1]}`); + } + for (const word of words) { + if (word.length < 4) continue; + for (let index = 0; index + 2 < word.length; index += 1) { + features.push(`#${word.slice(index, index + 3)}`); + } + } + return features; +} + +/** + * Dependency-free lexical hashing for unit tests and portable smoke runs. + * This adapter is not a semantic model. Measured runs should use + * `createOpenAITextEmbedder` with a recorded model and immutable revision. + */ +export class PortableHashingTextEmbedder implements TextEmbedder { + readonly dimensions: number; + + constructor(dimensions = DEFAULT_HASH_DIMENSIONS) { + this.dimensions = positiveInteger(dimensions, "dimensions"); + } + + async embed( + texts: readonly string[], + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) throw new DOMException("embedding aborted", "AbortError"); + return texts.map((text) => { + const counts = new Map(); + for (const feature of lexicalFeatures(text)) { + counts.set(feature, (counts.get(feature) ?? 0) + 1); + } + const vector = Array.from({ length: this.dimensions }).fill(0); + for (const [feature, count] of counts) { + const hash = fnv1a(feature); + const index = hash % this.dimensions; + const sign = (hash & 0x80_00_00_00) === 0 ? 1 : -1; + vector[index] += sign * (1 + Math.log(count)); + } + // Empty strings are rejected by callers; keep a defined vector for text + // made only of punctuation so the router can fail safely during L2 normalization. + return vector; + }); + } +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts new file mode 100644 index 0000000000..894de1182c --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts @@ -0,0 +1,309 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { performance } from "node:perf_hooks"; + +import { canonicalJson, type JsonValue, sha256Hex } from "./catalog"; +import { + buildDenseToolIndex, + COMPOSITIONAL_ROUTER_HINT_COUNT, + COMPOSITIONAL_ROUTER_TOP_K, + type CompositionalRoutingEvidence, + type CompositionalRoutingResult, + routeCompositionalToolsPaired, +} from "./compositional-tool-router"; +import { + COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + type CompositionalRoutingCasePrediction, + type CompositionalRoutingComparison, + compareCompositionalRoutingVariants, +} from "./compositional-tool-routing-acceptance"; +import { + createOpenAIChatTaskDecomposer, + createOpenAITextEmbedder, + type ModelUsageEvent, + PortableHashingTextEmbedder, +} from "./compositional-tool-routing-adapters"; + +const SAFE_ENV_NAME = /^[A-Z][A-Z0-9_]{0,127}$/u; + +export interface CompositionalRoutingRemoteModelConfig { + base_url: string; + model: string; + revision: string; + api_key_env?: string; + allow_remote?: boolean; +} + +export interface CompositionalRoutingAcceptanceConfig { + decomposer: CompositionalRoutingRemoteModelConfig; + embedding: + | { kind: "portable"; dimensions?: number } + | ({ kind: "openai" } & CompositionalRoutingRemoteModelConfig); + timeout_ms?: number; +} + +export interface CompositionalRoutingAcceptanceOutput { + schema_version: "nemoclaw.compositional_tool_routing_acceptance.v1"; + generated_at: string; + claim_eligible: false; + configuration: { + decomposer_model: string; + decomposer_revision: string; + embedding_kind: "portable" | "openai"; + embedding_model: string; + embedding_revision: string; + top_k: number; + hint_count: number; + temperature: 0; + }; + corpus: { + tool_count: number; + case_count: number; + expected_step_count: number; + tools_sha256: string; + cases_sha256: string; + }; + usage: { + decomposition: { + requests: number; + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + duration_ms: number; + }; + embedding: { + requests: number; + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + duration_ms: number; + }; + index_build_time_ms: number; + end_to_end_time_ms: number; + }; + comparison: CompositionalRoutingComparison; + cases: readonly { + case_id: string; + expected_step_count: number; + shared_initial_decomposition: true; + initial: { + disposition: CompositionalRoutingResult["disposition"]; + forwarded_tool_names: readonly string[]; + forwarded_tool_count: number; + evidence: CompositionalRoutingEvidence; + }; + refined: { + disposition: CompositionalRoutingResult["disposition"]; + forwarded_tool_names: readonly string[]; + forwarded_tool_count: number; + evidence: CompositionalRoutingEvidence; + }; + }[]; + acceptance_passed: boolean; + limitations: readonly string[]; +} + +function requiredText(value: unknown, label: string): string { + if (typeof value !== "string" || !value.trim() || /[\r\n]/u.test(value)) { + throw new TypeError(`${label} must be a non-empty single-line string`); + } + return value.trim(); +} + +function publicIdentity(value: unknown, label: string): string { + const text = requiredText(value, label); + if ( + text.length > 256 || + text.includes("://") || + /(?:token|secret|password|api[_-]?key)/iu.test(text) + ) { + throw new TypeError(`${label} is not a public-safe model identity`); + } + return text; +} + +function credential(name: string | undefined, label: string): string | undefined { + if (name === undefined) return undefined; + if (!SAFE_ENV_NAME.test(name)) + throw new TypeError(`${label} must be an environment variable name`); + const value = process.env[name]; + if (!value) throw new Error(`${label} is not set`); + return value; +} + +function remoteOptions( + config: CompositionalRoutingRemoteModelConfig, + timeoutMs: number, + usage: ModelUsageEvent[], +) { + return { + baseUrl: requiredText(config.base_url, "model base_url"), + model: publicIdentity(config.model, "model"), + apiKey: credential(config.api_key_env, "model api_key_env"), + allowRemote: config.allow_remote === true, + timeoutMs, + onUsage: (event: ModelUsageEvent) => usage.push(event), + }; +} + +function prediction( + caseId: string, + result: CompositionalRoutingResult, +): CompositionalRoutingCasePrediction { + return { + case_id: caseId, + disposition: result.disposition, + fallback: result.evidence.fallback, + steps: result.evidence.final_candidate_tool_names.map((ranked, index) => ({ + subtask: `private-step-${index + 1}`, + ranked_tool_names: ranked, + })), + selected_tool_names: result.selected_tool_names, + }; +} + +function observation(result: CompositionalRoutingResult) { + const forwardedToolNames = + result.disposition === "passthrough" + ? COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.map((tool) => tool.name) + : [...result.selected_tool_names]; + return { + disposition: result.disposition, + forwarded_tool_names: forwardedToolNames, + forwarded_tool_count: forwardedToolNames.length, + evidence: result.evidence, + }; +} + +function sumUsage( + events: readonly ModelUsageEvent[], + key: "prompt_tokens" | "completion_tokens" | "total_tokens", +) { + const values = events + .map((event) => event[key]) + .filter((value): value is number => value !== undefined); + return values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0); +} + +function operationUsage( + events: readonly ModelUsageEvent[], + operation: ModelUsageEvent["operation"], +): CompositionalRoutingAcceptanceOutput["usage"]["decomposition"] { + const selected = events.filter((event) => event.operation === operation); + const promptTokens = sumUsage(selected, "prompt_tokens"); + const completionTokens = sumUsage(selected, "completion_tokens"); + const totalTokens = sumUsage(selected, "total_tokens"); + return { + requests: selected.length, + ...(promptTokens === undefined ? {} : { prompt_tokens: promptTokens }), + ...(completionTokens === undefined ? {} : { completion_tokens: completionTokens }), + ...(totalTokens === undefined ? {} : { total_tokens: totalTokens }), + duration_ms: selected.reduce((total, event) => total + event.duration_ms, 0), + }; +} + +/** Run the public-safe route-only acceptance path with a real decomposer. */ +export async function runCompositionalRoutingAcceptance( + config: CompositionalRoutingAcceptanceConfig, +): Promise { + const timeoutMs = config.timeout_ms ?? 120_000; + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new TypeError("timeout_ms must be a positive safe integer"); + } + const usage: ModelUsageEvent[] = []; + const decomposerRevision = publicIdentity(config.decomposer.revision, "decomposer revision"); + const decomposer = createOpenAIChatTaskDecomposer( + remoteOptions(config.decomposer, timeoutMs, usage), + ); + const embeddingKind = config.embedding.kind; + const embeddingModel = + embeddingKind === "portable" + ? "portable-lexical-hashing" + : publicIdentity(config.embedding.model, "embedding model"); + const embeddingRevision = + embeddingKind === "portable" + ? "builtin-v1" + : publicIdentity(config.embedding.revision, "embedding revision"); + const embedder = + embeddingKind === "portable" + ? new PortableHashingTextEmbedder(config.embedding.dimensions) + : createOpenAITextEmbedder(remoteOptions(config.embedding, timeoutMs, usage)); + + const startedAt = performance.now(); + const indexStartedAt = performance.now(); + const preparedIndex = await buildDenseToolIndex(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, embedder); + const indexBuildTimeMs = performance.now() - indexStartedAt; + const initialPredictions: CompositionalRoutingCasePrediction[] = []; + const refinedPredictions: CompositionalRoutingCasePrediction[] = []; + const cases: CompositionalRoutingAcceptanceOutput["cases"][number][] = []; + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + const pair = await routeCompositionalToolsPaired( + fixture.prompt, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + { decomposer, embedder, preparedIndex }, + ); + initialPredictions.push(prediction(fixture.id, pair.initial)); + refinedPredictions.push(prediction(fixture.id, pair.refined)); + cases.push({ + case_id: fixture.id, + expected_step_count: fixture.expected_steps.length, + shared_initial_decomposition: true, + initial: observation(pair.initial), + refined: observation(pair.refined), + }); + } + const comparison = compareCompositionalRoutingVariants( + { variant: "initial", cases: initialPredictions }, + { variant: "refined", cases: refinedPredictions }, + ); + return { + schema_version: "nemoclaw.compositional_tool_routing_acceptance.v1", + generated_at: new Date().toISOString(), + claim_eligible: false, + configuration: { + decomposer_model: publicIdentity(config.decomposer.model, "decomposer model"), + decomposer_revision: decomposerRevision, + embedding_kind: embeddingKind, + embedding_model: embeddingModel, + embedding_revision: embeddingRevision, + top_k: COMPOSITIONAL_ROUTER_TOP_K, + hint_count: COMPOSITIONAL_ROUTER_HINT_COUNT, + temperature: 0, + }, + corpus: { + tool_count: COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.length, + case_count: COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.length, + expected_step_count: COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.reduce( + (total, fixture) => total + fixture.expected_steps.length, + 0, + ), + tools_sha256: sha256Hex( + canonicalJson(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS as unknown as JsonValue), + ), + cases_sha256: sha256Hex( + canonicalJson(COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES as unknown as JsonValue), + ), + }, + usage: { + decomposition: operationUsage(usage, "decomposition"), + embedding: operationUsage(usage, "embedding"), + index_build_time_ms: indexBuildTimeMs, + end_to_end_time_ms: performance.now() - startedAt, + }, + comparison, + cases, + acceptance_passed: comparison.passed, + limitations: [ + "This route-only acceptance run does not execute an agent or a tool.", + "The eight-case corpus is software acceptance coverage, not a universal quality result.", + ...(decomposerRevision === "unreported" + ? ["The decomposer revision was not reported, so this run is not reproducible evidence."] + : []), + ...(embeddingKind === "portable" + ? ["The portable lexical embedder is for smoke testing and is not claim eligible."] + : []), + ], + }; +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts new file mode 100644 index 0000000000..e3775fc473 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts @@ -0,0 +1,349 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { + buildDenseToolIndex, + type CompositionalRouterOptions, + type CompositionalRoutingEvidence, + type DenseToolIndex, + routeCompositionalTools, + type ToolRoutingCatalogEntry, +} from "./compositional-tool-router"; +import type { + RecorderEndpoint, + RecorderMethod, + RecordingProxyRequestTransform, + RecordingProxyTransformInput, +} from "./recorder"; + +const SAFE_TOOL_NAME = /^[A-Za-z0-9_-]{1,128}$/u; + +interface JsonRecord { + [key: string]: unknown; +} + +interface ParsedTool { + name: string; + description: string; + definition: unknown; + routable: boolean; +} + +interface CachedRoute { + selectedNames: ReadonlySet; + evidence: CompositionalTransformEvidence; +} + +export interface CompositionalTransformEvidence { + run_id: string; + source_tool_count: number; + routable_tool_count: number; + preserved_tool_count: number; + forwarded_tool_count: number; + cache_hits: number; + index_cache_hit: boolean; + transform_bypass: "tool-choice-conflict" | null; + routing: CompositionalRoutingEvidence; +} + +export interface CompositionalToolRoutingTransformOptions extends CompositionalRouterOptions { + /** + * Explicitly identify the reviewed catalog under test while retaining every + * framework, provider-native, and core tool. + */ + isRoutableTool: (name: string) => boolean; + /** Make unsupported/bypassed requests fail instead of silently acting as direct mode. */ + requireRouting?: boolean; +} + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function parsedTool( + definition: unknown, + isRoutableTool: (name: string) => boolean, +): ParsedTool | null { + if (!isRecord(definition) || definition.type !== "function") return null; + const nested = isRecord(definition.function) ? definition.function : null; + if (!nested) return null; + const rawName = nested.name; + const rawDescription = nested.description ?? ""; + if ( + typeof rawName !== "string" || + !SAFE_TOOL_NAME.test(rawName) || + typeof rawDescription !== "string" + ) { + return null; + } + return { + name: rawName, + description: rawDescription, + definition, + routable: isRoutableTool(rawName), + }; +} + +function textContent(value: unknown): string | null { + if (typeof value === "string" && value.trim()) return value; + if (!Array.isArray(value)) return null; + const parts = value.flatMap((part) => { + if (!isRecord(part)) return []; + const text = part.text; + return typeof text === "string" && text.trim() ? [text] : []; + }); + return parts.length > 0 ? parts.join("\n") : null; +} + +function extractChatQuery(payload: JsonRecord): string | null { + if (!Array.isArray(payload.messages)) return null; + for (let index = payload.messages.length - 1; index >= 0; index -= 1) { + const message = payload.messages[index]; + if (!isRecord(message) || message.role !== "user") continue; + const content = textContent(message.content); + if (content) return content; + } + return null; +} + +function catalogFingerprint(tools: readonly ParsedTool[]): string { + const hash = createHash("sha256"); + for (const tool of tools) { + hash.update(tool.name, "utf8"); + hash.update("\0", "utf8"); + hash.update(tool.description, "utf8"); + hash.update("\0", "utf8"); + hash.update(tool.routable ? "route" : "preserve", "utf8"); + hash.update("\0", "utf8"); + } + return hash.digest("hex"); +} + +function queryFingerprint(query: string): string { + return createHash("sha256").update(query, "utf8").digest("hex"); +} + +function namedToolChoice(payload: JsonRecord): string | null { + if (!isRecord(payload.tool_choice) || !isRecord(payload.tool_choice.function)) return null; + const name = payload.tool_choice.function.name; + return typeof name === "string" ? name : null; +} + +function cloneEvidence(value: CompositionalTransformEvidence): CompositionalTransformEvidence { + return structuredClone(value); +} + +function abortError(): DOMException { + return new DOMException("routing aborted", "AbortError"); +} + +/** + * Let each request stop waiting independently without cancelling shared index + * or route work that another live request may still need. + */ +function awaitWithAbort(pending: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(abortError()); + return new Promise((resolve, reject) => { + const onAbort = () => reject(abortError()); + signal.addEventListener("abort", onAbort, { once: true }); + pending.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (error: unknown) => { + signal.removeEventListener("abort", onAbort); + reject(error); + }, + ); + }); +} + +/** + * Connect the pure refined router to the recording proxy without changing + * an agent's executor registry. Only the forwarded model request is filtered. + */ +export class CompositionalToolRoutingTransform { + readonly requestTransform: RecordingProxyRequestTransform; + + private readonly isRoutableTool: (name: string) => boolean; + private readonly routes = new Map>(); + private readonly routeKeysByRun = new Map>(); + private readonly indexes = new Map>(); + + constructor(private readonly options: CompositionalToolRoutingTransformOptions) { + if (typeof options.isRoutableTool !== "function") { + throw new TypeError("compositional routing requires an explicit isRoutableTool predicate"); + } + this.isRoutableTool = options.isRoutableTool; + this.requestTransform = (input) => this.transform(input); + } + + async consumeEvidence(runId: string): Promise { + const keys = this.routeKeysByRun.get(runId); + if (!keys) return []; + this.routeKeysByRun.delete(runId); + const evidence: CompositionalTransformEvidence[] = []; + for (const key of keys) { + const pending = this.routes.get(key); + this.routes.delete(key); + if (!pending) continue; + try { + evidence.push(cloneEvidence((await pending).evidence)); + } catch { + // Aborted/failed routes do not retain partial evidence. + } + } + return evidence; + } + + private bypass(body: Buffer, reason: string): Buffer { + if (this.options.requireRouting) { + throw new Error(`required compositional routing bypassed: ${reason}`); + } + return body; + } + + private preparedIndex( + fingerprint: string, + catalog: readonly ToolRoutingCatalogEntry[], + ): { promise: Promise; hit: boolean } { + const existing = this.indexes.get(fingerprint); + if (existing) return { promise: existing, hit: true }; + // Cached work must not inherit one request's cancellation signal. + const promise = buildDenseToolIndex(catalog, this.options.embedder); + this.indexes.set(fingerprint, promise); + void promise.catch(() => { + if (this.indexes.get(fingerprint) === promise) this.indexes.delete(fingerprint); + }); + if (this.indexes.size > 32) { + const oldest = this.indexes.keys().next().value as string | undefined; + if (oldest && oldest !== fingerprint) this.indexes.delete(oldest); + } + return { promise, hit: false }; + } + + private shouldTransform( + endpoint: RecorderEndpoint, + method: RecorderMethod, + modelCallSequence: number | null, + runId: string | null, + ): runId is string { + return ( + endpoint === "chat-completions" && + method === "POST" && + modelCallSequence !== null && + runId !== null + ); + } + + private async transform(input: RecordingProxyTransformInput): Promise { + if (!this.shouldTransform(input.endpoint, input.method, input.modelCallSequence, input.runId)) { + return this.bypass(input.body, "unsupported request boundary"); + } + const runId = input.runId; + + let payload: JsonRecord; + try { + const value: unknown = JSON.parse(input.body.toString("utf8")); + if (!isRecord(value)) return this.bypass(input.body, "request body is not an object"); + payload = value; + } catch { + return this.bypass(input.body, "request body is not valid JSON"); + } + if (!Array.isArray(payload.tools) || payload.tools.length === 0) { + return this.bypass(input.body, "request has no tools"); + } + + const parsed = payload.tools.map((tool) => parsedTool(tool, this.isRoutableTool)); + // Preserve the whole request when any schema shape is unknown; never drop a + // tool the transformer cannot identify safely. + if (parsed.some((tool) => tool === null)) { + return this.bypass(input.body, "request contains an unsupported tool schema"); + } + const tools = parsed as ParsedTool[]; + const routable = tools.filter((tool) => tool.routable); + if (routable.length === 0) return this.bypass(input.body, "request has no routable tools"); + + const fingerprint = catalogFingerprint(tools); + const query = extractChatQuery(payload); + if (!query) return this.bypass(input.body, "request has no user query"); + const routeKey = `${runId}:${queryFingerprint(query)}:${fingerprint}`; + let pending = this.routes.get(routeKey); + const cacheHit = pending !== undefined; + + if (!pending) { + const catalog: ToolRoutingCatalogEntry[] = routable.map((tool) => ({ + name: tool.name, + description: tool.description, + definition: tool.definition, + })); + const prepared = this.preparedIndex(fingerprint, catalog); + pending = (async (): Promise => { + const result = await routeCompositionalTools(query, catalog, { + ...this.options, + preparedIndex: prepared.promise, + // Requests cancel only their own wait; shared work stays usable. + signal: undefined, + }); + const selected = + result.disposition === "passthrough" + ? new Set(routable.map((tool) => tool.name)) + : new Set(result.selected_tool_names); + if ( + result.selected_tool_names.some((name) => !routable.some((tool) => tool.name === name)) + ) { + throw new Error("router selected a tool outside the reviewed catalog"); + } + const forwardedToolCount = tools.filter( + (tool) => !tool.routable || selected.has(tool.name), + ).length; + return { + selectedNames: selected, + evidence: { + run_id: runId, + source_tool_count: tools.length, + routable_tool_count: routable.length, + preserved_tool_count: tools.length - routable.length, + forwarded_tool_count: + result.disposition === "passthrough" ? tools.length : forwardedToolCount, + cache_hits: 0, + index_cache_hit: prepared.hit, + transform_bypass: null, + routing: result.evidence, + }, + }; + })(); + this.routes.set(routeKey, pending); + const runKeys = this.routeKeysByRun.get(runId) ?? new Set(); + runKeys.add(routeKey); + this.routeKeysByRun.set(runId, runKeys); + void pending.catch(() => { + if (this.routes.get(routeKey) === pending) this.routes.delete(routeKey); + this.routeKeysByRun.get(runId)?.delete(routeKey); + }); + } + + const cached = await awaitWithAbort(pending, input.signal); + if (cacheHit) cached.evidence.cache_hits += 1; + + if (cached.evidence.routing.fallback !== null) { + return this.bypass(input.body, `routing fallback: ${cached.evidence.routing.fallback}`); + } + const choice = namedToolChoice(payload); + if ( + choice && + routable.some((tool) => tool.name === choice) && + !cached.selectedNames.has(choice) + ) { + cached.evidence.transform_bypass = "tool-choice-conflict"; + cached.evidence.forwarded_tool_count = tools.length; + return this.bypass(input.body, "named tool choice conflicts with routed catalog"); + } + payload.tools = tools + .filter((tool) => !tool.routable || cached.selectedNames.has(tool.name)) + .map((tool) => tool.definition); + return Buffer.from(JSON.stringify(payload), "utf8"); + } +} diff --git a/scripts/performance/tool-disclosure/recorder.ts b/scripts/performance/tool-disclosure/recorder.ts index 189f809c70..17712663db 100644 --- a/scripts/performance/tool-disclosure/recorder.ts +++ b/scripts/performance/tool-disclosure/recorder.ts @@ -92,6 +92,8 @@ export interface ToolDisclosureRecordingEvent { export interface RecordingProxyOptions { upstreamBaseUrl: string; + /** Explicitly allow a credential-free non-loopback HTTPS upstream. */ + allowRemoteHttpsUpstream?: boolean; /** Must be exactly 127.0.0.1. Exposed so unsafe configuration fails closed. */ listenHost?: string; /** Zero selects an ephemeral port. */ @@ -99,8 +101,30 @@ export interface RecordingProxyOptions { maxRequestBodyBytes?: number; requestTimeoutMs?: number; requiredTemperature?: number; + requestTransform?: RecordingProxyRequestTransform; } +/** + * Private request content supplied only to an opt-in transform invocation. + * + * The body is a defensive copy and is never retained by the proxy. Transform + * implementations must likewise avoid logging or retaining it. Public evidence + * continues to contain only the content-free metrics derived after transformation. + */ +export interface RecordingProxyTransformInput { + readonly runId: string | null; + readonly endpoint: RecorderEndpoint; + readonly method: RecorderMethod; + readonly modelCallSequence: number | null; + readonly body: Buffer; + readonly signal: AbortSignal; +} + +/** Return the complete request body to forward upstream. */ +export type RecordingProxyRequestTransform = ( + input: RecordingProxyTransformInput, +) => Buffer | Promise; + export interface RecordingProxyAddress { host: typeof LOOPBACK_HOST; port: number; @@ -148,8 +172,10 @@ interface ActiveRun { interface ForwardContext { bodyComplete: boolean; + transformComplete: boolean; cancelled: boolean; timedOut: boolean; + abortController: AbortController; upstreamRequest: http.ClientRequest | null; } @@ -171,7 +197,7 @@ function validatePositiveInteger(value: number, label: string): void { } } -function parseUpstreamBaseUrl(value: string): URL { +function parseUpstreamBaseUrl(value: string, allowRemoteHttpsUpstream: boolean): URL { let url: URL; try { url = new URL(value); @@ -188,9 +214,12 @@ function parseUpstreamBaseUrl(value: string): URL { hostname === "localhost" || hostname === "::1" || (isIP(hostname) === 4 && hostname.startsWith("127.")); - if (!loopback) { + if (!loopback && !allowRemoteHttpsUpstream) { throw new Error("upstreamBaseUrl is allowed only on loopback"); } + if (!loopback && url.protocol !== "https:") { + throw new Error("a remote recording upstream must use HTTPS"); + } if (url.username || url.password) { throw new Error("upstreamBaseUrl must not contain credentials"); } @@ -385,6 +414,7 @@ export class ToolDisclosureRecordingProxy { private readonly maxRequestBodyBytes: number; private readonly requestTimeoutMs: number; private readonly requiredTemperature: number | undefined; + private readonly requestTransform: RecordingProxyRequestTransform | undefined; private readonly clock: () => number; private readonly sockets = new Set(); private readonly events: ToolDisclosureRecordingEvent[] = []; @@ -406,12 +436,16 @@ export class ToolDisclosureRecordingProxy { validatePositiveInteger(maxRequestBodyBytes, "maxRequestBodyBytes"); validatePositiveInteger(requestTimeoutMs, "requestTimeoutMs"); - this.upstream = parseUpstreamBaseUrl(options.upstreamBaseUrl); + this.upstream = parseUpstreamBaseUrl( + options.upstreamBaseUrl, + options.allowRemoteHttpsUpstream === true, + ); this.listenHost = listenHost; this.configuredPort = port; this.maxRequestBodyBytes = maxRequestBodyBytes; this.requestTimeoutMs = requestTimeoutMs; this.requiredTemperature = options.requiredTemperature; + this.requestTransform = options.requestTransform; this.clock = () => performance.now(); } @@ -609,6 +643,30 @@ export class ToolDisclosureRecordingProxy { return target; } + private async transformRequestBody( + body: Buffer, + pending: PendingRecording | null, + endpoint: RecorderEndpoint, + method: RecorderMethod, + signal: AbortSignal, + ): Promise { + if (!this.requestTransform) return body; + const transformed = await this.requestTransform({ + runId: pending?.runId ?? null, + endpoint, + method, + modelCallSequence: pending?.modelCallSequence ?? null, + body: Buffer.from(body), + signal, + }); + if (!Buffer.isBuffer(transformed)) { + throw new TypeError("recording proxy request transform must return a Buffer"); + } + // Detach forwarding from any reference retained by the transform so later + // mutation cannot alter the bytes inspected, recorded, or sent upstream. + return Buffer.from(transformed); + } + private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise { let incoming: URL; try { @@ -627,10 +685,16 @@ export class ToolDisclosureRecordingProxy { const pending = this.reserveRecording(endpoint, method); const context: ForwardContext = { bodyComplete: false, + transformComplete: false, cancelled: false, timedOut: false, + abortController: new AbortController(), upstreamRequest: null, }; + request.once("aborted", () => context.abortController.abort()); + response.once("close", () => { + if (!response.writableEnded) context.abortController.abort(); + }); const declaredLength = Number(request.headers["content-length"]); if (Number.isFinite(declaredLength) && declaredLength > this.maxRequestBodyBytes) { @@ -647,20 +711,22 @@ export class ToolDisclosureRecordingProxy { const timer = setTimeout(() => { context.cancelled = true; context.timedOut = true; + context.abortController.abort(); if (context.upstreamRequest) { context.upstreamRequest.destroy(); return; } - const statusCode = context.bodyComplete ? 504 : 408; + const waitingForUpstream = context.bodyComplete && context.transformComplete; + const statusCode = waitingForUpstream ? 504 : 408; fixedResponse( response, statusCode, - context.bodyComplete ? "upstream timeout" : "request timeout", + waitingForUpstream ? "upstream timeout" : "request timeout", ); this.finalizeRecording(pending, { statusCode, - outcome: context.bodyComplete ? "upstream-timeout" : "request-rejected", - errorReason: context.bodyComplete ? "upstream-timeout" : "request-timeout", + outcome: waitingForUpstream ? "upstream-timeout" : "request-rejected", + errorReason: waitingForUpstream ? "upstream-timeout" : "request-timeout", }); request.destroy(); }, this.requestTimeoutMs); @@ -688,13 +754,31 @@ export class ToolDisclosureRecordingProxy { return; } - let forwardedBody = bodyResult.body; - if (pending) pending.metrics = inspectTools(forwardedBody); + let forwardedBody = await this.transformRequestBody( + bodyResult.body, + pending, + endpoint, + method, + context.abortController.signal, + ); + context.transformComplete = true; + if (context.cancelled) return; + if (forwardedBody.length > this.maxRequestBodyBytes) { + fixedResponse(response, 413, "request body too large"); + this.finalizeRecording(pending, { + statusCode: 413, + outcome: "request-rejected", + errorReason: "request-body-too-large", + }); + return; + } + let forwardedMetrics = inspectTools(forwardedBody); + if (pending) pending.metrics = forwardedMetrics; if ( pending?.modelCallSequence != null && this.requiredTemperature !== undefined && - pending.metrics.temperature !== null && - pending.metrics.temperature !== this.requiredTemperature + forwardedMetrics.temperature !== null && + forwardedMetrics.temperature !== this.requiredTemperature ) { fixedResponse(response, 400, "performance test temperature mismatch"); this.finalizeRecording(pending, { @@ -707,11 +791,22 @@ export class ToolDisclosureRecordingProxy { if ( pending?.modelCallSequence != null && this.requiredTemperature !== undefined && - pending.metrics.temperature === null + forwardedMetrics.temperature === null ) { const payload = JSON.parse(forwardedBody.toString("utf8")) as Record; payload.temperature = this.requiredTemperature; forwardedBody = Buffer.from(JSON.stringify(payload), "utf8"); + if (forwardedBody.length > this.maxRequestBodyBytes) { + fixedResponse(response, 413, "request body too large"); + this.finalizeRecording(pending, { + statusCode: 413, + outcome: "request-rejected", + errorReason: "request-body-too-large", + }); + return; + } + forwardedMetrics = inspectTools(forwardedBody); + if (pending) pending.metrics = forwardedMetrics; } await this.forwardRequest( request, diff --git a/scripts/performance/tool-disclosure/run.ts b/scripts/performance/tool-disclosure/run.ts index 7e0570c735..8ec35d7fa8 100644 --- a/scripts/performance/tool-disclosure/run.ts +++ b/scripts/performance/tool-disclosure/run.ts @@ -23,6 +23,10 @@ import { generateSyntheticCatalog, type SyntheticCatalog, } from "./catalog"; +import { + type CompositionalRoutingAcceptanceConfig, + runCompositionalRoutingAcceptance, +} from "./compositional-tool-routing-run"; import { type AttemptJournalEntry, type CampaignAttestation, @@ -58,7 +62,7 @@ import { import { validateCompleteEvidence, validateFrozenManifest } from "./validation"; interface ParsedArguments { - command: "prepare" | "execute" | "summarize" | "help"; + command: "prepare" | "execute" | "summarize" | "route-acceptance" | "help"; outputDir?: string; sutRef: string; catalogSeed: string; @@ -90,6 +94,7 @@ function usage(): string { Usage:\n\ npm run performance:tool-disclosure -- prepare --output-dir [options]\n\ npm run performance:tool-disclosure -- execute --output-dir --config \n\ + npm run performance:tool-disclosure -- route-acceptance --output-dir --config \n\ npm run performance:tool-disclosure -- summarize --output-dir [options]\n\n\ Prepare options:\n\ --sut-ref SUT revision to resolve (default: HEAD)\n\ @@ -112,7 +117,12 @@ function requiredValue(args: readonly string[], index: number, flag: string): st function parseArguments(args: readonly string[]): ParsedArguments { const first = args[0]; const command = - first === "prepare" || first === "execute" || first === "summarize" ? first : "help"; + first === "prepare" || + first === "execute" || + first === "summarize" || + first === "route-acceptance" + ? first + : "help"; if (first && command === "help" && first !== "help" && first !== "--help" && first !== "-h") { throw new Error(`unknown command: ${first}`); } @@ -153,8 +163,9 @@ function parseArguments(args: readonly string[]): ParsedArguments { } else throw new Error(`unknown option: ${flag}`); } if (parsed.command !== "help" && !parsed.outputDir) throw new Error("--output-dir is required"); - if (parsed.command === "execute" && !parsed.configPath) - throw new Error("execute requires --config"); + if (["execute", "route-acceptance"].includes(parsed.command) && !parsed.configPath) { + throw new Error(`${parsed.command} requires --config`); + } return parsed; } @@ -705,6 +716,27 @@ async function execute(options: ParsedArguments): Promise { }); } +async function routeAcceptance(options: ParsedArguments): Promise { + const outputDir = ensureCampaignDirectory( + path.resolve(options.outputDir as string), + options.resume, + ); + const config = readJson( + path.resolve(options.configPath as string), + ); + const output = await runCompositionalRoutingAcceptance(config); + const artifactName = "compositional-routing-acceptance.json"; + writeJsonArtifact(outputDir, artifactName, output); + scanArtifactsForForbiddenValues(outputDir, [artifactName], options.forbiddenValues); + writeChecksumManifest(outputDir, [artifactName]); + process.stdout.write( + `Compositional routing acceptance ${output.acceptance_passed ? "passed" : "failed"}.\n`, + ); + if (!output.acceptance_passed) { + throw new Error("compositional routing acceptance gates did not pass"); + } +} + export async function main(args = process.argv.slice(2)): Promise { const options = parseArguments(args); if (options.command === "help") { @@ -713,6 +745,7 @@ export async function main(args = process.argv.slice(2)): Promise { } if (options.command === "prepare") prepare(options); else if (options.command === "execute") await execute(options); + else if (options.command === "route-acceptance") await routeAcceptance(options); else summarize(options); } diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 2af313d868..39246c9835 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -9,6 +9,12 @@ import { generateCatalogPrefix, generateSyntheticCatalog, } from "../../../scripts/performance/tool-disclosure/catalog"; +import { + createOpenAIChatTaskDecomposer, + PortableHashingTextEmbedder, +} from "../../../scripts/performance/tool-disclosure/compositional-tool-routing-adapters"; +import { runCompositionalRoutingAcceptance } from "../../../scripts/performance/tool-disclosure/compositional-tool-routing-run"; +import { CompositionalToolRoutingTransform } from "../../../scripts/performance/tool-disclosure/compositional-tool-routing-transform"; import { buildAgentDriverCommand, extractFinalAssistantOutput, @@ -16,6 +22,7 @@ import { import { gradeTaskRun } from "../../../scripts/performance/tool-disclosure/grading"; import { SyntheticMcpServer } from "../../../scripts/performance/tool-disclosure/mcp-server"; import { startQuickTunnel } from "../../../scripts/performance/tool-disclosure/quick-tunnel"; +import { createToolDisclosureRecordingProxy } from "../../../scripts/performance/tool-disclosure/recorder"; import type { ToolDisclosureMode } from "../../../scripts/performance/tool-disclosure/schedule"; import { generatePrimaryTaskSet } from "../../../scripts/performance/tool-disclosure/tasks"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -64,6 +71,28 @@ test.skipIf(!shouldRunLiveE2E())( const bearerToken = randomBytes(32).toString("hex"); artifacts.addRedactionValues([hosted.apiKey, bearerToken]); + const routingCredentialEnv = "NEMOCLAW_COMPOSITIONAL_ROUTING_SMOKE_API_KEY"; + const previousRoutingCredential = process.env[routingCredentialEnv]; + process.env[routingCredentialEnv] = hosted.apiKey; + let routingAcceptance: Awaited>; + try { + routingAcceptance = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: hosted.endpointUrl, + model: hosted.model, + revision: process.env.NEMOCLAW_MODEL_REVISION?.trim() || "unreported", + api_key_env: routingCredentialEnv, + allow_remote: true, + }, + embedding: { kind: "portable" }, + }); + } finally { + if (previousRoutingCredential === undefined) delete process.env[routingCredentialEnv]; + else process.env[routingCredentialEnv] = previousRoutingCredential; + } + await artifacts.writeJson("compositional-routing-acceptance.json", routingAcceptance); + expect(routingAcceptance.acceptance_passed).toBe(true); + const catalog = generateSyntheticCatalog({ seed: DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, }); @@ -220,6 +249,165 @@ test.skipIf(!shouldRunLiveE2E())( }); } + const routedRunId = `performance-smoke-${AGENT}-compositional-${TASK_ID}`; + const routedToolNames = new Set( + catalogPrefix.tools.map((tool) => `${MCP_SERVER_NAME}_${tool.definition.function.name}`), + ); + const expectedRoutedToolName = `${MCP_SERVER_NAME}_${task.expected_calls[0]?.tool_name ?? ""}`; + if (!routedToolNames.has(expectedRoutedToolName)) { + throw new Error("the routed replay target is not present in the reviewed catalog"); + } + const routedTransform = new CompositionalToolRoutingTransform({ + decomposer: createOpenAIChatTaskDecomposer({ + baseUrl: hosted.endpointUrl, + model: hosted.model, + apiKey: hosted.apiKey, + allowRemote: true, + }), + embedder: new PortableHashingTextEmbedder(), + isRoutableTool: (name) => routedToolNames.has(name), + requireRouting: true, + }); + const routedProxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: hosted.endpointUrl, + allowRemoteHttpsUpstream: true, + requestTransform: routedTransform.requestTransform, + }); + const routedProxyAddress = await routedProxy.start(); + cleanup.add("stop compositional routing recording proxy", () => routedProxy.stop()); + + const configureInferenceRoute = async (baseUrl: string, artifactSuffix: string) => { + const update = await host.command( + "openshell", + [ + "provider", + "update", + hosted.providerName, + "--credential", + hosted.credentialEnv, + "--config", + `OPENAI_BASE_URL=${baseUrl}`, + ], + { + artifactName: `compositional-routing-provider-${artifactSuffix}`, + env: hostEnv(hosted.env), + redactionValues: [hosted.apiKey], + timeoutMs: 2 * 60_000, + }, + ); + expect(update.exitCode, resultText(update)).toBe(0); + const select = await host.command( + "openshell", + [ + "inference", + "set", + "--no-verify", + "--provider", + hosted.providerName, + "--model", + hosted.model, + ], + { + artifactName: `compositional-routing-select-${artifactSuffix}`, + env: hostEnv(hosted.env), + redactionValues: [hosted.apiKey], + timeoutMs: 2 * 60_000, + }, + ); + expect(select.exitCode, resultText(select)).toBe(0); + }; + + let routedReplay: Record | undefined; + let routeMutationAttempted = false; + try { + routeMutationAttempted = true; + await configureInferenceRoute(`${routedProxyAddress.base_url}/v1`, "enable"); + const driver = buildAgentDriverCommand({ + openshellBin: process.env.OPENSHELL_BIN, + sandboxName: sandboxName("direct"), + agent: AGENT, + prompt: task.prompt, + sessionId: routedRunId, + }); + mcp.beginRun(routedRunId); + routedProxy.beginRun(routedRunId); + const startedAt = process.hrtime.bigint(); + let invocation; + let calls; + let recorderEvents; + try { + invocation = await host.command(driver.command, driver.args, { + artifactName: "invoke-tool-disclosure-compositional", + env: hostEnv(), + redactionValues: [...driver.redactions, hosted.apiKey, bearerToken], + timeoutMs: 10 * 60_000, + }); + } finally { + calls = mcp.endRun(); + recorderEvents = routedProxy.endRun(); + } + const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); + const routeEvidence = await routedTransform.consumeEvidence(routedRunId); + const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); + const graded = gradeTaskRun(task, calls, finalOutput); + const outcome = invocation.timedOut + ? "timeout" + : invocation.exitCode === 0 + ? graded.outcome + : "model-error"; + routedReplay = { + schema_version: "nemoclaw.compositional_tool_routing_agent_smoke.v1", + generated_at: new Date().toISOString(), + claim_eligible: false, + run_id: routedRunId, + outcome, + invocation: { + exit_code: invocation.exitCode, + timed_out: invocation.timedOut, + elapsed_ms: elapsedMs, + }, + synthetic_call_count: calls.length, + correctness: graded.correctness, + expected_routed_tool_name: expectedRoutedToolName, + route_evidence: routeEvidence, + recorder_events: recorderEvents, + limitations: [ + "This single routed replay is an end-to-end wiring check, not a performance or quality claim.", + "The portable lexical embedder is used only for an ordinary-runner smoke path.", + "The routed replay is separate from the frozen direct/progressive comparison.", + ], + }; + } finally { + if (routeMutationAttempted) { + await configureInferenceRoute(hosted.endpointUrl, "restore"); + } + } + + await artifacts.writeJson("compositional-routing-agent-smoke.json", routedReplay); + expect(routedReplay).toBeDefined(); + const replay = routedReplay as { + outcome: string; + correctness: { task_success: boolean }; + synthetic_call_count: number; + route_evidence: Awaited>; + recorder_events: Array<{ + model_call_sequence: number | null; + visible_tool_count: number; + }>; + }; + expect(replay.outcome).toBe("success"); + expect(replay.correctness.task_success).toBe(true); + expect(replay.synthetic_call_count).toBe(1); + expect(replay.route_evidence.length).toBeGreaterThan(0); + const [firstRoute] = replay.route_evidence; + expect(firstRoute.routing.fallback).toBeNull(); + expect(firstRoute.transform_bypass).toBeNull(); + expect(firstRoute.routable_tool_count).toBe(CATALOG_SIZE); + expect(firstRoute.forwarded_tool_count).toBeLessThan(firstRoute.source_tool_count); + expect(firstRoute.routing.selected_tool_names).toContain(expectedRoutedToolName); + const firstModelEvent = replay.recorder_events.find((event) => event.model_call_sequence === 1); + expect(firstModelEvent?.visible_tool_count).toBe(firstRoute.forwarded_tool_count); + await artifacts.writeJson("tool-disclosure-performance-smoke.json", { schema_version: "nemoclaw.tool_disclosure_performance_smoke.v1", generated_at: new Date().toISOString(), diff --git a/test/performance/tool-disclosure-compositional-router.test.ts b/test/performance/tool-disclosure-compositional-router.test.ts new file mode 100644 index 0000000000..2823e1791d --- /dev/null +++ b/test/performance/tool-disclosure-compositional-router.test.ts @@ -0,0 +1,411 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + buildDenseToolIndex, + type DecompositionRequest, + exactInnerProductTopK, + l2Normalize, + routeCompositionalTools, + routeCompositionalToolsPaired, + selectRoutedToolDefinitions, + type TaskDecomposer, + type TextEmbedder, + type ToolRoutingCatalogEntry, + unionToolHints, +} from "../../scripts/performance/tool-disclosure/compositional-tool-router"; + +interface TestDefinition { + type: "function"; + function: { + name: string; + parameters: { type: "object"; properties: Record }; + }; +} + +function definition(name: string): TestDefinition { + return { + type: "function", + function: { + name, + parameters: { type: "object", properties: { value: { type: "string" } } }, + }, + }; +} + +function catalogEntry(name: string, description: string): ToolRoutingCatalogEntry { + return { name, description, definition: definition(name) }; +} + +class KeywordEmbedder implements TextEmbedder { + readonly calls: string[][] = []; + + async embed(texts: readonly string[]): Promise { + this.calls.push([...texts]); + return texts.map((text) => { + const normalized = text.toLowerCase(); + if (/calendar|appointment|event/u.test(normalized)) return [3, 0, 0, 0]; + if (/email|notify|message/u.test(normalized)) return [0, 4, 0, 0]; + if (/weather|forecast/u.test(normalized)) return [0, 0, 5, 0]; + return [0, 0, 0, 2]; + }); + } +} + +describe("independent compositional tool dense retrieval", () => { + it("normalizes vectors and applies exact top-k with score/name tie-breaking", async () => { + expect(l2Normalize([3, 4])).toEqual([0.6, 0.8]); + expect(() => l2Normalize([0, 0])).toThrow("positive finite L2 norm"); + expect(() => l2Normalize([1, Number.NaN])).toThrow("must be finite"); + + const catalog = [ + catalogEntry("zeta", "first horizontal tool"), + catalogEntry("alpha", "second horizontal tool"), + catalogEntry("vertical", "vertical tool"), + ]; + const embedder: TextEmbedder = { + async embed() { + return [ + [20, 0], + [2, 0], + [0, 7], + ]; + }, + }; + const index = await buildDenseToolIndex(catalog, embedder); + + expect(index.tools.map((tool) => tool.vector)).toEqual([ + [1, 0], + [1, 0], + [0, 1], + ]); + expect(exactInnerProductTopK(index, [9, 0], 2)).toEqual([ + { name: "alpha", score: 1 }, + { name: "zeta", score: 1 }, + ]); + + const tied = { + dimension: 1, + tools: Array.from({ length: 20 }, (_, toolIndex) => ({ + name: `tool-${String(toolIndex).padStart(2, "0")}`, + vector: [1], + })), + }; + expect(exactInnerProductTopK(tied, [1])).toHaveLength(10); + }); + + it("unions hints by maximum score before applying the deterministic cap", () => { + const hints = unionToolHints( + [ + [ + { name: "shared", score: 0.2 }, + { name: "bravo", score: 0.8 }, + { name: "charlie", score: 0.7 }, + ], + [ + { name: "shared", score: 0.9 }, + { name: "alpha", score: 0.8 }, + ], + ], + 3, + ); + + expect(hints).toEqual([ + { name: "shared", score: 0.9 }, + { name: "alpha", score: 0.8 }, + { name: "bravo", score: 0.8 }, + ]); + + const defaultHints = unionToolHints([ + Array.from({ length: 10 }, (_, index) => ({ name: `left-${index}`, score: 1 - index / 100 })), + Array.from({ length: 10 }, (_, index) => ({ + name: `right-${index}`, + score: 0.5 - index / 100, + })), + ]); + expect(defaultHints).toHaveLength(12); + }); +}); + +describe("independent compositional tool routing", () => { + it("shares one initial decomposition and one prepared index across paired routes", async () => { + const catalog = [ + catalogEntry("calendar_find", "Find an appointment in a calendar."), + catalogEntry("email_send", "Send an email message to notify a teammate."), + ]; + const decompose = vi.fn(async (request) => + request.pass === "initial" + ? ["locate the appointment", "notify the teammate"] + : ["find the calendar event", "send the email message"], + ); + const embedder = new KeywordEmbedder(); + + const pair = await routeCompositionalToolsPaired("Arrange and notify.", catalog, { + decomposer: { decompose }, + embedder, + }); + + expect(pair.shared_initial_decomposition).toBe(true); + expect(pair.shared_initial_subtask_count).toBe(2); + expect(decompose.mock.calls.filter(([request]) => request.pass === "initial")).toHaveLength(1); + expect(decompose.mock.calls.filter(([request]) => request.pass === "refined")).toHaveLength(1); + expect( + embedder.calls.filter((batch) => batch.every((text) => text.includes("\n"))), + ).toHaveLength(1); + expect(embedder.calls).toHaveLength(3); + expect(embedder.calls[1]).toEqual(["locate the appointment", "notify the teammate"]); + expect(embedder.calls[2]).toEqual(["find the calendar event", "send the email message"]); + expect(pair.initial.evidence.initial_candidate_tool_names).toEqual( + pair.refined.evidence.initial_candidate_tool_names, + ); + expect(pair.initial.evidence.timings.initial_decomposition_ms).toBe( + pair.refined.evidence.timings.initial_decomposition_ms, + ); + expect(pair.initial.evidence.timings.catalog_embedding_ms).toBe( + pair.refined.evidence.timings.catalog_embedding_ms, + ); + expect(pair.initial.evidence.timings.initial_retrieval_ms).toBe( + pair.refined.evidence.timings.initial_retrieval_ms, + ); + expect(pair.initial.selected_tool_names).toEqual(pair.refined.selected_tool_names); + }); + + it("shares paired initial failures without starting retrieval or refinement", async () => { + const decompose = vi.fn().mockResolvedValue({ malformed: true }); + const embed = vi.fn(); + const pair = await routeCompositionalToolsPaired( + "Arrange and notify.", + [catalogEntry("calendar_find", "Find a calendar event.")], + { decomposer: { decompose }, embedder: { embed } }, + ); + + expect(decompose).toHaveBeenCalledOnce(); + expect(embed).not.toHaveBeenCalled(); + expect(pair.initial).toMatchObject({ + disposition: "passthrough", + evidence: { fallback: "initial-decomposition-malformed" }, + }); + expect(pair.refined).toMatchObject({ + disposition: "passthrough", + evidence: { fallback: "initial-decomposition-malformed" }, + }); + expect(pair.initial.evidence.timings.initial_decomposition_ms).toBe( + pair.refined.evidence.timings.initial_decomposition_ms, + ); + }); + + it("shares a paired no-tool result without embedding or refining", async () => { + const decompose = vi.fn().mockResolvedValue([]); + const embed = vi.fn(); + const pair = await routeCompositionalToolsPaired( + "Answer without tools.", + [catalogEntry("calendar_find", "Find a calendar event.")], + { decomposer: { decompose }, embedder: { embed } }, + ); + + expect(decompose).toHaveBeenCalledOnce(); + expect(embed).not.toHaveBeenCalled(); + expect(pair.initial).toMatchObject({ + disposition: "routed", + selected_tool_names: [], + evidence: { decomposition_passes: 1, fallback: null }, + }); + expect(pair.refined).toMatchObject({ + disposition: "routed", + selected_tool_names: [], + evidence: { decomposition_passes: 1, fallback: null }, + }); + }); + + it("supports a one-pass initial route over the same retriever", async () => { + const catalog = [ + catalogEntry("calendar_find", "Find an appointment in a calendar."), + catalogEntry("email_send", "Send an email message to notify a teammate."), + catalogEntry("weather_read", "Read a weather forecast."), + ]; + const decompose = vi + .fn() + .mockResolvedValue(["find the calendar event", "send the email message"]); + + const result = await routeCompositionalTools("Arrange and notify.", catalog, { + decomposer: { decompose }, + embedder: new KeywordEmbedder(), + refinementPasses: 0, + }); + + expect(decompose).toHaveBeenCalledOnce(); + expect(result.disposition).toBe("routed"); + expect(result.selected_tool_names).toEqual(["calendar_find", "email_send"]); + expect(result.evidence).toMatchObject({ + initial_subtask_count: 2, + refined_subtask_count: 0, + decomposition_passes: 1, + hint_count: 0, + selected_tool_count: 2, + fallback: null, + }); + expect(result.evidence.final_candidate_counts).toEqual([3, 3]); + }); + + it("runs one initial pass and one hint-informed refinement before bounded selection", async () => { + const catalog = [ + catalogEntry("calendar_find", "Find an appointment in a calendar."), + catalogEntry("email_send", "Send an email message to notify a teammate."), + catalogEntry("weather_read", "Read a weather forecast."), + catalogEntry("generic_lookup", "Look up an unrelated record."), + ]; + const requests: DecompositionRequest[] = []; + const decomposer: TaskDecomposer = { + async decompose(request) { + requests.push(request); + return request.pass === "initial" + ? ["locate the appointment", "notify the teammate"] + : ["find the calendar event", "send the email message"]; + }, + }; + const embedder = new KeywordEmbedder(); + const result = await routeCompositionalTools( + "Arrange the appointment, then tell the teammate.", + catalog, + { decomposer, embedder, maxSelectedTools: 2 }, + ); + + expect(requests).toHaveLength(2); + expect(requests[0]).toEqual({ + pass: "initial", + query: "Arrange the appointment, then tell the teammate.", + tool_hints: [], + }); + expect(requests[1].pass).toBe("refined"); + expect(requests[1].tool_hints).toContain("calendar_find"); + expect(requests[1].tool_hints).toContain("email_send"); + expect(result.disposition).toBe("routed"); + expect(result.selected_tool_names).toEqual(["calendar_find", "email_send"]); + expect(result.evidence).toMatchObject({ + initial_subtask_count: 2, + refined_subtask_count: 2, + decomposition_passes: 2, + selected_tool_count: 2, + selected_tool_names: ["calendar_find", "email_send"], + fallback: null, + }); + expect(result.evidence.initial_candidate_counts).toEqual([4, 4]); + expect(result.evidence.final_candidate_counts).toEqual([4, 4]); + expect(result.evidence.final_candidate_tool_names[0][0]).toBe("calendar_find"); + expect(result.evidence.final_candidate_tool_names[1][0]).toBe("email_send"); + expect(JSON.stringify(result.evidence)).not.toContain("Arrange the appointment"); + expect(JSON.stringify(result.evidence)).not.toContain("locate the appointment"); + expect(embedder.calls).toHaveLength(3); + }); + + it("fails open for malformed decomposition and unavailable retrieval", async () => { + const catalog = [ + catalogEntry("calendar_find", "Find a calendar event."), + catalogEntry("email_send", "Send an email message."), + ]; + const malformed = await routeCompositionalTools("do work", catalog, { + decomposer: { + async decompose() { + return { steps: ["do work"] }; + }, + }, + embedder: new KeywordEmbedder(), + }); + expect(malformed.disposition).toBe("passthrough"); + expect(malformed.evidence.fallback).toBe("initial-decomposition-malformed"); + expect(selectRoutedToolDefinitions(catalog, malformed)).toEqual( + catalog.map((entry) => entry.definition), + ); + + const unavailable = await routeCompositionalTools("find the appointment", catalog, { + decomposer: { + async decompose() { + return ["find the calendar event"]; + }, + }, + embedder: { + async embed() { + throw new Error("embedding service unavailable"); + }, + }, + }); + expect(unavailable.disposition).toBe("passthrough"); + expect(unavailable.evidence.fallback).toBe("catalog-embedding-failed"); + expect(unavailable.selected_tool_names).toEqual([]); + + const disagreement = await routeCompositionalTools("find the appointment", catalog, { + decomposer: { + async decompose(request) { + return request.pass === "initial" ? ["find the calendar event"] : []; + }, + }, + embedder: new KeywordEmbedder(), + }); + expect(disagreement.disposition).toBe("passthrough"); + expect(disagreement.evidence.fallback).toBe("refinement-no-tool-disagreement"); + }); + + it("treats an empty initial decomposition as a valid no-tool route", async () => { + const embed = vi.fn(); + const decompose = vi.fn().mockResolvedValue([]); + const catalog = [catalogEntry("calendar_find", "Find a calendar event.")]; + + const result = await routeCompositionalTools("Answer without tools.", catalog, { + decomposer: { decompose }, + embedder: { embed }, + }); + + expect(result.disposition).toBe("routed"); + expect(result.selected_tool_names).toEqual([]); + expect(result.evidence).toMatchObject({ + initial_subtask_count: 0, + refined_subtask_count: 0, + decomposition_passes: 1, + hint_count: 0, + selected_tool_count: 0, + fallback: null, + }); + expect(decompose).toHaveBeenCalledOnce(); + expect(embed).not.toHaveBeenCalled(); + expect(selectRoutedToolDefinitions(catalog, result)).toEqual([]); + }); + + it("returns original schema objects in selected order and fails open above the cap", async () => { + const calendar = catalogEntry("calendar_find", "Find a calendar event."); + const email = catalogEntry("email_send", "Send an email message."); + const catalog = [email, calendar]; + const originalSchemas = structuredClone(catalog.map((entry) => entry.definition)); + const decomposer: TaskDecomposer = { + async decompose(request) { + return request.pass === "initial" + ? ["calendar event", "email message"] + : ["find calendar event", "send email message"]; + }, + }; + const result = await routeCompositionalTools("schedule and notify", catalog, { + decomposer, + embedder: new KeywordEmbedder(), + maxSelectedTools: 2, + }); + const selected = selectRoutedToolDefinitions(catalog, result); + + expect(selected).toEqual([calendar.definition, email.definition]); + expect(selected[0]).toBe(calendar.definition); + expect(selected[1]).toBe(email.definition); + expect(catalog.map((entry) => entry.definition)).toEqual(originalSchemas); + + const capped = await routeCompositionalTools("schedule and notify", catalog, { + decomposer, + embedder: new KeywordEmbedder(), + maxSelectedTools: 1, + }); + expect(capped.disposition).toBe("passthrough"); + expect(capped.evidence.fallback).toBe("selection-limit-exceeded"); + const passthrough = selectRoutedToolDefinitions(catalog, capped); + expect(passthrough).toEqual([email.definition, calendar.definition]); + expect(passthrough[0]).toBe(email.definition); + expect(passthrough[1]).toBe(calendar.definition); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-routing-acceptance.test.ts b/test/performance/tool-disclosure-compositional-routing-acceptance.test.ts new file mode 100644 index 0000000000..a3aeba26ff --- /dev/null +++ b/test/performance/tool-disclosure-compositional-routing-acceptance.test.ts @@ -0,0 +1,353 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + routeCompositionalTools, + routeCompositionalToolsPaired, +} from "../../scripts/performance/tool-disclosure/compositional-tool-router"; +import { + COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + type CompositionalRoutingCasePrediction, + type CompositionalRoutingVariant, + type CompositionalRoutingVariantInput, + compareCompositionalRoutingVariants, + DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS, + evaluateCompositionalRoutingGates, + evaluateCompositionalRoutingVariant, +} from "../../scripts/performance/tool-disclosure/compositional-tool-routing-acceptance"; +import { PortableHashingTextEmbedder } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-adapters"; + +const toolNames = COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.map((tool) => tool.name); + +function perfectPrediction(variant: CompositionalRoutingVariant): CompositionalRoutingVariantInput { + return { + variant, + cases: COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.map((fixture) => { + const expectedNames = fixture.expected_steps.map((expected) => expected.tool_name); + return { + case_id: fixture.id, + disposition: "routed" as const, + fallback: null, + steps: fixture.expected_steps.map((expected, index) => ({ + subtask: `${fixture.id} scripted atomic step ${index + 1}`, + ranked_tool_names: [ + expected.tool_name, + ...toolNames.filter((name) => name !== expected.tool_name), + ], + })), + selected_tool_names: expectedNames, + }; + }), + }; +} + +function replaceCase( + input: CompositionalRoutingVariantInput, + caseId: string, + update: (prediction: CompositionalRoutingCasePrediction) => CompositionalRoutingCasePrediction, +): CompositionalRoutingVariantInput { + return { + ...input, + cases: input.cases.map((prediction) => + prediction.case_id === caseId ? update(prediction) : prediction, + ), + }; +} + +describe("compositional routing acceptance corpus", () => { + it("contains distinct metadata and unambiguous cases from zero through five steps", () => { + expect(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS).toHaveLength(20); + expect(new Set(toolNames).size).toBe(toolNames.length); + expect( + new Set(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.map((tool) => tool.description)).size, + ).toBe(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.length); + + const stepCounts = COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.map( + (fixture) => fixture.expected_steps.length, + ); + expect(new Set(stepCounts)).toEqual(new Set([0, 1, 2, 3, 4, 5])); + expect(stepCounts.reduce((total, count) => total + count, 0)).toBe(18); + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + expect(fixture.prompt.trim()).not.toBe(""); + for (const expected of fixture.expected_steps) { + expect(toolNames).toContain(expected.tool_name); + expect(fixture.prompt.toLocaleLowerCase("en-US")).toContain(expected.prompt_cue); + expect(fixture.prompt).not.toContain(expected.tool_name); + } + } + }); + + it("scores perfect scripted decomposition, retrieval, selection, and no-tool behavior", () => { + const metrics = evaluateCompositionalRoutingVariant(perfectPrediction("refined")); + + expect(metrics).toMatchObject({ + variant: "refined", + case_count: 8, + expected_step_count: 18, + predicted_step_count: 18, + decomposition_exact_rate: 1, + decomposition_within_one_rate: 1, + exact_tool_recall_at_1: 1, + exact_tool_recall_at_10: 1, + chain_exact_selection_rate: 1, + selection_count_exact_rate: 1, + selected_tool_count: 18, + mean_selected_tool_count: 2.25, + max_selected_tool_count: 5, + no_tool_case_count: 1, + no_tool_exact_rate: 1, + }); + expect(metrics.cases.flatMap((fixture) => fixture.expected_tool_ranks)).toEqual( + Array.from({ length: 18 }, () => 1), + ); + expect(evaluateCompositionalRoutingGates(metrics)).toMatchObject({ passed: true, reasons: [] }); + }); + + it("passes the acceptance gates through the real router when decomposition is held fixed", async () => { + const embedder = new PortableHashingTextEmbedder(); + const predictions: CompositionalRoutingCasePrediction[] = []; + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + const subtasks = fixture.expected_steps.map((expected) => expected.capability); + const routed = await routeCompositionalTools( + fixture.prompt, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + { + embedder, + decomposer: { decompose: async () => subtasks }, + }, + ); + expect(routed.disposition, fixture.id).toBe("routed"); + predictions.push({ + case_id: fixture.id, + disposition: routed.disposition, + fallback: routed.evidence.fallback, + steps: routed.evidence.final_candidate_tool_names.map((ranked, index) => ({ + subtask: subtasks[index] ?? "no external capability", + ranked_tool_names: ranked, + })), + selected_tool_names: routed.selected_tool_names, + }); + } + + const metrics = evaluateCompositionalRoutingVariant({ variant: "refined", cases: predictions }); + expect(metrics).toMatchObject({ + decomposition_exact_rate: 1, + exact_tool_recall_at_1: 1, + exact_tool_recall_at_10: 1, + chain_exact_selection_rate: 1, + selection_count_exact_rate: 1, + max_selected_tool_count: 5, + no_tool_exact_rate: 1, + }); + expect(evaluateCompositionalRoutingGates(metrics)).toMatchObject({ passed: true, reasons: [] }); + }); + + it("shares the initial model output in the scripted initial-versus-refined control", async () => { + const embedder = new PortableHashingTextEmbedder(); + const initialCases: CompositionalRoutingCasePrediction[] = []; + const refinedCases: CompositionalRoutingCasePrediction[] = []; + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + const refinedSubtasks = fixture.expected_steps.map((expected) => expected.capability); + const initialSubtasks = + refinedSubtasks.length <= 1 + ? refinedSubtasks + : [`complete the ${refinedSubtasks.length} requested operations`]; + const pair = await routeCompositionalToolsPaired( + fixture.prompt, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + { + embedder, + decomposer: { + decompose: async (request) => + request.pass === "refined" ? refinedSubtasks : initialSubtasks, + }, + }, + ); + expect(pair.shared_initial_decomposition, fixture.id).toBe(true); + initialCases.push({ + case_id: fixture.id, + disposition: pair.initial.disposition, + fallback: pair.initial.evidence.fallback, + steps: pair.initial.evidence.final_candidate_tool_names.map((ranked, index) => ({ + subtask: initialSubtasks[index] ?? "no external capability", + ranked_tool_names: ranked, + })), + selected_tool_names: pair.initial.selected_tool_names, + }); + refinedCases.push({ + case_id: fixture.id, + disposition: pair.refined.disposition, + fallback: pair.refined.evidence.fallback, + steps: pair.refined.evidence.final_candidate_tool_names.map((ranked, index) => ({ + subtask: refinedSubtasks[index] ?? "no external capability", + ranked_tool_names: ranked, + })), + selected_tool_names: pair.refined.selected_tool_names, + }); + } + + const comparison = compareCompositionalRoutingVariants( + { variant: "initial", cases: initialCases }, + { variant: "refined", cases: refinedCases }, + ); + + expect(comparison.initial.decomposition_exact_rate).toBeLessThan(1); + expect(comparison.refined.decomposition_exact_rate).toBe(1); + expect(comparison.refined.exact_tool_recall_at_1).toBe(1); + expect(comparison.refined_minus_initial.decomposition_exact_rate).toBeGreaterThan(0); + expect(comparison.refined_not_worse).toBe(true); + expect(comparison.refined_gate.passed).toBe(true); + expect(comparison.passed).toBe(true); + }); + + it("compares scripted initial and refined inputs without treating them as empirical evidence", () => { + let initial = perfectPrediction("initial"); + initial = replaceCase(initial, "route-chain-three-analysis-01", (prediction) => ({ + ...prediction, + steps: prediction.steps.slice(0, 2), + selected_tool_names: prediction.selected_tool_names.slice(0, 2), + })); + initial = replaceCase(initial, "route-single-bars-01", (prediction) => ({ + ...prediction, + steps: prediction.steps.map((step) => ({ + ...step, + ranked_tool_names: [ + "route_render_line_chart", + ...step.ranked_tool_names.filter((name) => name !== "route_render_line_chart"), + ], + })), + selected_tool_names: ["route_render_line_chart"], + })); + + const comparison = compareCompositionalRoutingVariants(initial, perfectPrediction("refined")); + + expect(comparison.initial.decomposition_exact_rate).toBeLessThan(1); + expect(comparison.initial.exact_tool_recall_at_1).toBeLessThan(1); + expect(comparison.initial.chain_exact_selection_rate).toBeLessThan(1); + expect(comparison.refined_minus_initial.decomposition_exact_rate).toBeGreaterThan(0); + expect(comparison.refined_minus_initial.exact_tool_recall_at_1).toBeGreaterThan(0); + expect(comparison.refined_not_worse).toBe(true); + expect(comparison.refined_gate.passed).toBe(true); + expect(comparison.passed).toBe(true); + }); + + it("fails strict gates when one selected chain is wrong", () => { + const refined = replaceCase( + perfectPrediction("refined"), + "route-single-bars-01", + (prediction) => ({ + ...prediction, + selected_tool_names: ["route_render_line_chart"], + }), + ); + const metrics = evaluateCompositionalRoutingVariant(refined); + const gate = evaluateCompositionalRoutingGates(metrics); + + expect(metrics.exact_tool_recall_at_1).toBe(1); + expect(metrics.chain_exact_selection_rate).toBe(7 / 8); + expect(gate.passed).toBe(false); + expect(gate.reasons).toContain("chain_exact_selection_rate 0.875 is below 1"); + }); + + it("counts a no-tool decomposition or selection as a control failure", () => { + const refined = replaceCase(perfectPrediction("refined"), "route-no-tool-01", (prediction) => ({ + ...prediction, + steps: [ + { + subtask: "unnecessary external lookup", + ranked_tool_names: ["route_lookup_directory_contact"], + }, + ], + selected_tool_names: ["route_lookup_directory_contact"], + })); + const metrics = evaluateCompositionalRoutingVariant(refined); + + expect(metrics.no_tool_exact_rate).toBe(0); + expect(metrics.selection_count_exact_rate).toBe(7 / 8); + expect(metrics.max_selected_tool_count).toBe(5); + expect(evaluateCompositionalRoutingGates(metrics).passed).toBe(false); + }); + + it("cannot score a failed-open no-tool route as successful", () => { + const refined = replaceCase(perfectPrediction("refined"), "route-no-tool-01", (prediction) => ({ + ...prediction, + disposition: "passthrough", + fallback: "initial-decomposition-failed", + selected_tool_names: [], + })); + const metrics = evaluateCompositionalRoutingVariant(refined); + + expect(metrics.routing_success_rate).toBe(7 / 8); + expect(metrics.route_failure_case_count).toBe(1); + expect(metrics.no_tool_exact_rate).toBe(0); + expect(metrics.chain_exact_forwarding_rate).toBe(7 / 8); + expect(metrics.forwarding_count_exact_rate).toBe(7 / 8); + expect(metrics.max_forwarded_tool_count).toBe(20); + expect(metrics.cases[0]).toMatchObject({ + disposition: "passthrough", + routing_succeeded: false, + forwarded_tool_count: 20, + no_tool_exact: false, + }); + const permissive = Object.fromEntries( + Object.entries(DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS).map(([key, value]) => [ + key, + key.startsWith("min_") ? 0 : Math.max(value, 20), + ]), + ) as unknown as typeof DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS; + expect(evaluateCompositionalRoutingGates(metrics, permissive)).toMatchObject({ + passed: false, + reasons: ["route_failure_case_count 1 must be zero"], + }); + }); + + it("rejects incomplete, duplicate, and unknown-tool evidence", () => { + const perfect = perfectPrediction("refined"); + expect(() => + evaluateCompositionalRoutingVariant({ ...perfect, cases: perfect.cases.slice(1) }), + ).toThrow(/exactly 8 cases/); + expect(() => + evaluateCompositionalRoutingVariant({ + ...perfect, + cases: [perfect.cases[0], ...perfect.cases.slice(0, -1)], + }), + ).toThrow(/duplicate case/); + expect(() => + evaluateCompositionalRoutingVariant( + replaceCase(perfect, "route-single-csv-01", (prediction) => ({ + ...prediction, + selected_tool_names: ["route_unknown"], + })), + ), + ).toThrow(/selected unknown tool/); + expect(() => + evaluateCompositionalRoutingVariant( + replaceCase(perfect, "route-single-csv-01", (prediction) => ({ + ...prediction, + steps: prediction.steps.map((step) => ({ + ...step, + ranked_tool_names: [step.ranked_tool_names[0], step.ranked_tool_names[0]], + })), + })), + ), + ).toThrow(/ranking contains duplicates/); + expect(() => + evaluateCompositionalRoutingVariant( + replaceCase(perfect, "route-single-csv-01", (prediction) => ({ + ...prediction, + fallback: "initial-decomposition-failed", + })), + ), + ).toThrow(/inconsistent disposition and fallback/); + expect(() => + evaluateCompositionalRoutingVariant( + replaceCase(perfect, "route-single-csv-01", (prediction) => ({ + ...prediction, + disposition: "passthrough", + })), + ), + ).toThrow(/inconsistent disposition and fallback/); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-routing-adapters.test.ts b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts new file mode 100644 index 0000000000..8fa6fb4dd4 --- /dev/null +++ b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + createOpenAIChatTaskDecomposer, + createOpenAITextEmbedder, + type ModelUsageEvent, + PortableHashingTextEmbedder, +} from "../../scripts/performance/tool-disclosure/compositional-tool-routing-adapters"; + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("independent compositional tool model adapters", () => { + it("requests both decomposition passes without copying request text into usage evidence", async () => { + const responses = [["collect the records"], ["fetch records", "send a notification"]]; + const calls: Array<{ url: string; headers: Headers; body: Record }> = []; + const fetchMock = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { + calls.push({ + url: String(url), + headers: new Headers(init?.headers), + body: JSON.parse(String(init?.body)) as Record, + }); + const content = JSON.stringify(responses[calls.length - 1]); + return new Response( + JSON.stringify({ + choices: [{ message: { content: `\`\`\`json\n${content}\n\`\`\`` } }], + usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test/v1", + model: "test-model", + allowRemote: true, + apiKey: "private-api-key", + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ + pass: "initial", + query: "private request text", + tool_hints: [], + }), + ).resolves.toEqual(responses[0]); + await expect( + decomposer.decompose({ + pass: "refined", + query: "private request text", + tool_hints: ["route_fetch", "route_notify"], + }), + ).resolves.toEqual(responses[1]); + + expect(calls).toHaveLength(2); + expect(calls[0].url).toBe("https://models.example.test/v1/chat/completions"); + expect(calls[0].headers.get("authorization")).toBe("Bearer private-api-key"); + expect(calls[0].body).toMatchObject({ + model: "test-model", + temperature: 0, + max_tokens: 256, + stream: false, + }); + expect(JSON.stringify(calls[1].body)).toContain("route_fetch"); + expect(usage).toHaveLength(2); + expect(usage.map((event) => event.pass)).toEqual(["initial", "refined"]); + expect(usage[0]).toMatchObject({ + operation: "decomposition", + prompt_tokens: 12, + completion_tokens: 4, + total_tokens: 16, + }); + expect(JSON.stringify(usage)).not.toMatch(/private request|private-api-key|test-model/); + }); + + it("returns a fixed decomposition failure without relaying provider content", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("provider-private-error", { status: 500 })), + ); + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).rejects.toThrow("decomposition request failed"); + }); + + it("batches embeddings and restores provider results to input order", async () => { + const observedInputs: string[][] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const payload = JSON.parse(String(init?.body)) as { input: string[] }; + observedInputs.push(payload.input); + const data = payload.input + .map((text, index) => ({ index, embedding: [text.length, index + 1] })) + .reverse(); + return new Response(JSON.stringify({ data, usage: { total_tokens: 3 } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }), + ); + const usage: ModelUsageEvent[] = []; + const embedder = createOpenAITextEmbedder({ + baseUrl: "https://models.example.test/v1", + model: "test-embedding-model", + allowRemote: true, + batchSize: 2, + onUsage: (event) => usage.push(event), + }); + + await expect(embedder.embed(["a", "bb", "ccc"])).resolves.toEqual([ + [1, 1], + [2, 2], + [3, 1], + ]); + expect(observedInputs).toEqual([["a", "bb"], ["ccc"]]); + expect(usage).toHaveLength(2); + expect(usage.every((event) => event.operation === "embedding")).toBe(true); + }); + + it("provides a deterministic dependency-free lexical adapter for smoke tests", async () => { + const embedder = new PortableHashingTextEmbedder(512); + const [first, second, different] = await embedder.embed([ + "send an email notification", + "email notification send", + "decompress a gzip archive", + ]); + const dot = (left: readonly number[], right: readonly number[]) => + left.reduce((sum, value, index) => sum + value * right[index], 0); + + expect(first).toEqual(await embedder.embed(["send an email notification"]).then(([v]) => v)); + expect(dot(first, second)).toBeGreaterThan(dot(first, different)); + }); + + it("requires HTTPS and explicit opt-in before sending content off-host", () => { + expect(() => + createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test/v1", + model: "test-model", + }), + ).toThrow("require allowRemote"); + expect(() => + createOpenAITextEmbedder({ + baseUrl: "http://models.example.test/v1", + model: "test-model", + allowRemote: true, + }), + ).toThrow("must use HTTPS"); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-routing-run.test.ts b/test/performance/tool-disclosure-compositional-routing-run.test.ts new file mode 100644 index 0000000000..a918105235 --- /dev/null +++ b/test/performance/tool-disclosure-compositional-routing-run.test.ts @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-acceptance"; +import { runCompositionalRoutingAcceptance } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-run"; + +afterEach(() => { + delete process.env.ROUTING_TEST_API_KEY; + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("compositional routing acceptance runner", () => { + it("runs a shared-initial paired evaluation and emits public-safe evidence", async () => { + const privateKey = "private-route-acceptance-key"; + process.env.ROUTING_TEST_API_KEY = privateKey; + const fetchMock = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const payload = JSON.parse(String(init?.body)) as { + messages: Array<{ role: string; content: string }>; + }; + const user = + [...payload.messages].reverse().find((message) => message.role === "user")?.content ?? ""; + const fixture = COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.find((candidate) => + user.includes(candidate.prompt), + ); + if (!fixture) return new Response("missing fixture", { status: 400 }); + return new Response( + JSON.stringify({ + choices: [ + { + message: { + content: JSON.stringify( + fixture.expected_steps.map((expected) => expected.capability), + ), + }, + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + + const output = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + api_key_env: "ROUTING_TEST_API_KEY", + allow_remote: true, + }, + embedding: { kind: "portable", dimensions: 1_024 }, + }); + + expect(output).toMatchObject({ + schema_version: "nemoclaw.compositional_tool_routing_acceptance.v1", + claim_eligible: false, + configuration: { + decomposer_model: "test-decomposer", + decomposer_revision: "immutable-test-revision", + embedding_kind: "portable", + embedding_model: "portable-lexical-hashing", + embedding_revision: "builtin-v1", + top_k: 10, + hint_count: 12, + temperature: 0, + }, + corpus: { tool_count: 20, case_count: 8, expected_step_count: 18 }, + usage: { + decomposition: { + requests: 15, + prompt_tokens: 150, + completion_tokens: 30, + total_tokens: 180, + }, + embedding: { requests: 0 }, + }, + acceptance_passed: true, + }); + expect(output.cases).toHaveLength(8); + expect(output.cases.every((entry) => entry.shared_initial_decomposition)).toBe(true); + const noTool = output.cases.find((entry) => entry.case_id === "route-no-tool-01"); + expect(noTool?.refined).toMatchObject({ + disposition: "routed", + forwarded_tool_names: [], + forwarded_tool_count: 0, + evidence: { fallback: null }, + }); + const toolCase = output.cases.find((entry) => entry.case_id === "route-single-csv-01"); + expect(toolCase?.refined.forwarded_tool_names).toEqual( + toolCase?.refined.evidence.selected_tool_names, + ); + expect(output.comparison.refined_gate.passed).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(15); + const serialized = JSON.stringify(output); + expect(serialized).not.toContain(privateKey); + expect(serialized).not.toContain("models.example.test"); + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + expect(serialized).not.toContain(fixture.prompt); + } + }); + + it("fails acceptance and records full-catalog forwarding when no-tool routing fails open", async () => { + process.env.ROUTING_TEST_API_KEY = "private-route-acceptance-key"; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const payload = JSON.parse(String(init?.body)) as { + messages: Array<{ role: string; content: string }>; + }; + const user = + [...payload.messages].reverse().find((message) => message.role === "user")?.content ?? ""; + const fixture = COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.find((candidate) => + user.includes(candidate.prompt), + ); + if (!fixture) return new Response("missing fixture", { status: 400 }); + const content = + fixture.id === "route-no-tool-01" + ? "not-json" + : JSON.stringify(fixture.expected_steps.map((expected) => expected.capability)); + return new Response(JSON.stringify({ choices: [{ message: { content } }] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }), + ); + + const output = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + api_key_env: "ROUTING_TEST_API_KEY", + allow_remote: true, + }, + embedding: { kind: "portable" }, + }); + + expect(output.acceptance_passed).toBe(false); + expect(output.comparison.refined.route_failure_case_count).toBe(1); + expect(output.comparison.refined_gate.reasons).toContain( + "route_failure_case_count 1 must be zero", + ); + const noTool = output.cases.find((entry) => entry.case_id === "route-no-tool-01"); + expect(noTool?.refined).toMatchObject({ + disposition: "passthrough", + forwarded_tool_count: 20, + evidence: { fallback: "initial-decomposition-failed" }, + }); + expect(noTool?.refined.forwarded_tool_names).toHaveLength(20); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-routing-transform.test.ts b/test/performance/tool-disclosure-compositional-routing-transform.test.ts new file mode 100644 index 0000000000..b90cd65d8a --- /dev/null +++ b/test/performance/tool-disclosure-compositional-routing-transform.test.ts @@ -0,0 +1,407 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import http from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, describe, expect, it } from "vitest"; +import type { + DecompositionRequest, + TaskDecomposer, + TextEmbedder, +} from "../../scripts/performance/tool-disclosure/compositional-tool-router"; +import { CompositionalToolRoutingTransform } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-transform"; +import { createToolDisclosureRecordingProxy } from "../../scripts/performance/tool-disclosure/recorder"; + +function vectorFor(text: string): number[] { + const normalized = text.toLowerCase(); + if (normalized.includes("calendar")) return [1, 0, 0]; + if (normalized.includes("email")) return [0, 1, 0]; + return [0, 0, 1]; +} + +const embedder: TextEmbedder = { + embed: async (texts) => texts.map(vectorFor), +}; + +const cleanup: Array<() => Promise> = []; + +afterEach(async () => { + await Promise.allSettled(cleanup.splice(0).map((close) => close())); +}); + +function tools(): unknown[] { + return [ + { + type: "function", + function: { name: "core_shell", description: "Run a shell command.", parameters: {} }, + }, + { + type: "function", + function: { + name: "route_calendar_list", + description: "List calendar events.", + parameters: {}, + }, + }, + { + type: "function", + function: { + name: "route_storage_delete", + description: "Delete an object from storage.", + parameters: {}, + }, + }, + { + type: "function", + function: { + name: "route_email_send", + description: "Send an email message.", + parameters: {}, + }, + }, + ]; +} + +function requestBody(prompt = "List today's calendar events and send them by email."): Buffer { + return Buffer.from( + JSON.stringify({ + model: "private-model", + messages: [{ role: "user", content: prompt }], + tools: tools(), + }), + "utf8", + ); +} + +function input(body: Buffer, modelCallSequence = 1) { + return { + runId: "route-test-run", + endpoint: "chat-completions" as const, + method: "POST" as const, + modelCallSequence, + body, + signal: new AbortController().signal, + }; +} + +function names(body: Buffer): string[] { + const payload = JSON.parse(body.toString("utf8")) as { tools: unknown[] }; + return payload.tools.map((tool) => { + const value = tool as { function: { name: string } }; + return value.function.name; + }); +} + +describe("independent compositional tool request transform", () => { + it("filters the schemas measured and forwarded by the recording proxy", async () => { + let upstreamBody = ""; + const upstream = http.createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => { + upstreamBody = Buffer.concat(chunks).toString("utf8"); + response.end("{}"); + }); + }); + await new Promise((resolve, reject) => { + upstream.once("error", reject); + upstream.listen(0, "127.0.0.1", resolve); + }); + cleanup.push(() => new Promise((resolve) => upstream.close(() => resolve()))); + const address = upstream.address() as AddressInfo; + const transform = new CompositionalToolRoutingTransform({ + decomposer: { + decompose: async () => ["list calendar events", "send an email message"], + }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const proxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: `http://127.0.0.1:${address.port}`, + requestTransform: transform.requestTransform, + }); + const proxyAddress = await proxy.start(); + cleanup.push(() => proxy.stop()); + + proxy.beginRun("proxy-route-run"); + const response = await fetch(`${proxyAddress.base_url}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: requestBody(), + }); + await response.arrayBuffer(); + const [event] = proxy.endRun(); + + expect(response.status).toBe(200); + expect(names(Buffer.from(upstreamBody))).toEqual([ + "core_shell", + "route_calendar_list", + "route_email_send", + ]); + expect(event).toMatchObject({ + visible_tool_count: 3, + tool_names: ["core_shell", "route_calendar_list", "route_email_send"], + }); + expect(await transform.consumeEvidence("proxy-route-run")).toHaveLength(1); + }); + + it("runs two decomposition passes, filters only routable tools, and caches the route", async () => { + const requests: DecompositionRequest[] = []; + const decomposer: TaskDecomposer = { + decompose: async (request) => { + requests.push(request); + return ["list calendar events", "send an email message"]; + }, + }; + const transform = new CompositionalToolRoutingTransform({ + decomposer, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + + const body = requestBody(); + const first = await transform.requestTransform(input(body)); + const second = await transform.requestTransform(input(body, 2)); + + expect(names(first)).toEqual(["core_shell", "route_calendar_list", "route_email_send"]); + expect(second.equals(first)).toBe(true); + expect(requests).toHaveLength(2); + expect(requests.map((request) => request.pass)).toEqual(["initial", "refined"]); + expect(requests[1].tool_hints).toContain("route_calendar_list"); + expect(requests[1].tool_hints).toContain("route_email_send"); + + const [evidence] = await transform.consumeEvidence("route-test-run"); + expect(evidence).toMatchObject({ + source_tool_count: 4, + routable_tool_count: 3, + preserved_tool_count: 1, + forwarded_tool_count: 3, + cache_hits: 1, + index_cache_hit: false, + transform_bypass: null, + routing: { + initial_subtask_count: 2, + refined_subtask_count: 2, + decomposition_passes: 2, + selected_tool_count: 2, + selected_tool_names: ["route_calendar_list", "route_email_send"], + fallback: null, + }, + }); + expect(JSON.stringify(evidence)).not.toContain("today's calendar"); + expect(await transform.consumeEvidence("route-test-run")).toEqual([]); + }); + + it("keeps the full request byte-for-byte when routing falls back", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ({ malformed: true }) }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const body = requestBody("private fallback prompt"); + + const forwarded = await transform.requestTransform(input(body)); + + expect(forwarded.equals(body)).toBe(true); + const [evidence] = await transform.consumeEvidence("route-test-run"); + expect(evidence).toMatchObject({ + forwarded_tool_count: 4, + routing: { + decomposition_passes: 1, + fallback: "initial-decomposition-malformed", + }, + }); + expect(JSON.stringify(evidence)).not.toContain("private fallback prompt"); + }); + + it("rejects a routing fallback in strict mode while retaining its evidence", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ({ malformed: true }) }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + requireRouting: true, + }); + + await expect(transform.requestTransform(input(requestBody()))).rejects.toThrow( + "required compositional routing bypassed: routing fallback: initial-decomposition-malformed", + ); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { routing: { fallback: "initial-decomposition-malformed" } }, + ]); + }); + + it("treats an empty decomposition as an intentional no-tool route", async () => { + let embeddingCalls = 0; + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => [] }, + embedder: { + embed: async () => { + embeddingCalls += 1; + return []; + }, + }, + isRoutableTool: (name) => name.startsWith("route_"), + }); + + const forwarded = await transform.requestTransform( + input(requestBody("Reply with the literal phrase CONTROL and use no tools.")), + ); + + expect(names(forwarded)).toEqual(["core_shell"]); + expect(embeddingCalls).toBe(1); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { + forwarded_tool_count: 1, + routing: { + initial_subtask_count: 0, + refined_subtask_count: 0, + decomposition_passes: 1, + selected_tool_count: 0, + fallback: null, + }, + }, + ]); + }); + + it("leaves unsupported endpoints untouched and does not retain evidence", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ["unused"] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const body = requestBody(); + const forwarded = await transform.requestTransform({ + ...input(body), + endpoint: "responses", + }); + + expect(forwarded.equals(body)).toBe(true); + expect(await transform.consumeEvidence("route-test-run")).toEqual([]); + }); + + it("reuses the prepared catalog index across run IDs", async () => { + const embeddingBatchSizes: number[] = []; + const transform = new CompositionalToolRoutingTransform({ + decomposer: { + decompose: async () => ["list calendar events", "send an email message"], + }, + embedder: { + embed: async (texts) => { + embeddingBatchSizes.push(texts.length); + return texts.map(vectorFor); + }, + }, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const body = requestBody(); + + await transform.requestTransform(input(body)); + await transform.requestTransform({ ...input(body), runId: "route-test-run-2" }); + + const [first] = await transform.consumeEvidence("route-test-run"); + const [second] = await transform.consumeEvidence("route-test-run-2"); + expect(first.index_cache_hit).toBe(false); + expect(second.index_cache_hit).toBe(true); + expect(embeddingBatchSizes).toEqual([3, 2, 2, 2, 2]); + }); + + it("lets one request abort without cancelling shared route or index work", async () => { + let releaseInitial: ((subtasks: string[]) => void) | undefined; + let markInitialStarted: (() => void) | undefined; + const initialStarted = new Promise((resolve) => { + markInitialStarted = resolve; + }); + const decompositionSignals: Array = []; + const embeddingSignals: Array = []; + const transform = new CompositionalToolRoutingTransform({ + decomposer: { + decompose: async (request) => { + decompositionSignals.push(request.signal); + if (request.pass === "refined") return ["list calendar events"]; + markInitialStarted?.(); + return new Promise((resolve) => { + releaseInitial = resolve; + }); + }, + }, + embedder: { + embed: async (texts, signal) => { + embeddingSignals.push(signal); + return texts.map(vectorFor); + }, + }, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const firstController = new AbortController(); + const secondController = new AbortController(); + const body = requestBody("List today's calendar events."); + const first = transform.requestTransform({ + ...input(body), + signal: firstController.signal, + }); + await initialStarted; + const second = transform.requestTransform({ + ...input(body, 2), + signal: secondController.signal, + }); + const firstRejection = expect(first).rejects.toMatchObject({ name: "AbortError" }); + + firstController.abort(); + await firstRejection; + releaseInitial?.(["list calendar events"]); + + expect(names(await second)).toEqual(["core_shell", "route_calendar_list"]); + expect(decompositionSignals).toEqual([undefined, undefined]); + expect(embeddingSignals.every((signal) => signal === undefined)).toBe(true); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { cache_hits: 1, routing: { fallback: null } }, + ]); + }); + + it("preserves a request whose named tool choice conflicts with the route", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ["list calendar events"] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const payload = JSON.parse(requestBody().toString("utf8")) as Record; + payload.tool_choice = { + type: "function", + function: { name: "route_storage_delete" }, + }; + const body = Buffer.from(JSON.stringify(payload), "utf8"); + + const forwarded = await transform.requestTransform(input(body)); + + expect(forwarded.equals(body)).toBe(true); + const [evidence] = await transform.consumeEvidence("route-test-run"); + expect(evidence).toMatchObject({ + forwarded_tool_count: 4, + transform_bypass: "tool-choice-conflict", + }); + }); + + it("rejects a named tool-choice conflict in strict mode", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ["list calendar events"] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + requireRouting: true, + }); + const payload = JSON.parse(requestBody().toString("utf8")) as Record; + payload.tool_choice = { + type: "function", + function: { name: "route_storage_delete" }, + }; + + await expect( + transform.requestTransform(input(Buffer.from(JSON.stringify(payload), "utf8"))), + ).rejects.toThrow( + "required compositional routing bypassed: named tool choice conflicts with routed catalog", + ); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { transform_bypass: "tool-choice-conflict", forwarded_tool_count: 4 }, + ]); + }); +}); diff --git a/test/performance/tool-disclosure-recorder.test.ts b/test/performance/tool-disclosure-recorder.test.ts index 8f44fe6af9..c77973def1 100644 --- a/test/performance/tool-disclosure-recorder.test.ts +++ b/test/performance/tool-disclosure-recorder.test.ts @@ -7,8 +7,10 @@ import type { AddressInfo, Socket } from "node:net"; import { afterEach, describe, expect, it } from "vitest"; +import { canonicalJson, type JsonValue } from "../../scripts/performance/tool-disclosure/catalog"; import { createToolDisclosureRecordingProxy, + type RecordingProxyRequestTransform, type ToolDisclosureRecordingProxy, } from "../../scripts/performance/tool-disclosure/recorder"; @@ -55,6 +57,7 @@ async function startProxy( maxRequestBodyBytes?: number; requestTimeoutMs?: number; requiredTemperature?: number; + requestTransform?: RecordingProxyRequestTransform; } = {}, ): Promise<{ proxy: ToolDisclosureRecordingProxy; baseUrl: string }> { const proxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl, ...options }); @@ -114,6 +117,21 @@ describe("tool-disclosure recording proxy", () => { } }); + it("requires an explicit opt-in and HTTPS for a remote recording upstream", () => { + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + }), + ).not.toThrow(); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://inference.example/v1", + allowRemoteHttpsUpstream: true, + }), + ).toThrow("a remote recording upstream must use HTTPS"); + }); + it("canonicalizes the IPv6 loopback and rejects mapped or non-loopback variants", () => { expect(() => createToolDisclosureRecordingProxy({ @@ -295,6 +313,220 @@ describe("tool-disclosure recording proxy", () => { expect(proxy.getEvents()).toEqual([]); }); + it("applies an async request transform before forwarding and records transformed tools", async () => { + let receivedBody = ""; + const upstream = await startUpstream((request, response) => { + void collectBody(request).then((body) => { + receivedBody = body; + response.end("{}"); + }); + }); + const observedContexts: Array<{ + runId: string | null; + endpoint: string; + method: string; + modelCallSequence: number | null; + body: string; + }> = []; + const requestTransform: RecordingProxyRequestTransform = async (input) => { + observedContexts.push({ + runId: input.runId, + endpoint: input.endpoint, + method: input.method, + modelCallSequence: input.modelCallSequence, + body: input.body.toString("utf8"), + }); + const payload = JSON.parse(input.body.toString("utf8")) as Record; + const tools = payload.tools as unknown[]; + payload.tools = [tools[1]]; + await Promise.resolve(); + return Buffer.from(JSON.stringify(payload), "utf8"); + }; + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { requestTransform }); + const requestBody = JSON.stringify({ + model: "private-model", + messages: [{ role: "user", content: "private-prompt" }], + tools: [ + { + type: "function", + function: { name: "first_tool", description: "first-private", parameters: {} }, + }, + { + type: "function", + function: { name: "second_tool", description: "second-private", parameters: {} }, + }, + ], + }); + + const runId = proxy.beginRun("transform-run"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: requestBody, + }); + expect(response.status).toBe(200); + await response.arrayBuffer(); + + const [event] = proxy.endRun(); + const expectedPayload = JSON.parse(requestBody) as Record; + expectedPayload.tools = [(expectedPayload.tools as unknown[])[1]]; + const expectedBody = JSON.stringify(expectedPayload); + const canonicalTools = canonicalJson(expectedPayload.tools as JsonValue); + expect(receivedBody).toBe(expectedBody); + expect(observedContexts).toEqual([ + { + runId, + endpoint: "chat-completions", + method: "POST", + modelCallSequence: 1, + body: requestBody, + }, + ]); + expect(event).toMatchObject({ + visible_tool_count: 1, + tool_names: ["second_tool"], + canonical_tools_json_bytes: Buffer.byteLength(canonicalTools), + tools_sha256: createHash("sha256").update(canonicalTools).digest("hex"), + status_code: 200, + outcome: "completed", + }); + expect(proxy.consumeToolSchemaSnapshots(runId)).toEqual([ + { + run_id: runId, + model_call_sequence: 1, + canonical_tools_json: canonicalTools, + }, + ]); + expect(JSON.stringify(event)).not.toMatch( + /private-model|private-prompt|first-private|second-private/, + ); + }); + + it("forwards request bytes unchanged when no transform is configured", async () => { + let receivedBody = Buffer.alloc(0); + let receivedLength = ""; + const upstream = await startUpstream((request, response) => { + receivedLength = String(request.headers["content-length"] ?? ""); + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => { + receivedBody = Buffer.concat(chunks); + response.end("{}"); + }); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl); + const exactBody = Buffer.from( + '{\n "messages" : [{"content":"byte-private"}],\n "tools" : []\n}\n', + "utf8", + ); + + proxy.beginRun("no-transform-bytes"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: exactBody, + }); + await response.arrayBuffer(); + proxy.endRun(); + + expect(response.status).toBe(200); + expect(receivedBody.equals(exactBody)).toBe(true); + expect(receivedLength).toBe(String(exactBody.length)); + }); + + it("rejects transform failures with a fixed content-free proxy error", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTransform: async ({ body }) => { + expect(body.toString("utf8")).toContain("transform-input-private"); + throw new Error("transform-error-private"); + }, + }); + + const runId = proxy.beginRun("transform-error"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[{"content":"transform-input-private"}]}', + }); + expect(response.status).toBe(502); + expect(await response.json()).toEqual({ error: "recording proxy failure" }); + expect(hits).toBe(0); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + visible_tool_count: 0, + status_code: 502, + outcome: "request-rejected", + error_reason: "proxy-failure", + }); + expect(proxy.consumeToolSchemaSnapshots(runId)).toEqual([]); + expect(JSON.stringify(event)).not.toMatch(/transform-input-private|transform-error-private/); + }); + + it("rejects a transformed body that exceeds the request size bound", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + maxRequestBodyBytes: 32, + requestTransform: () => Buffer.alloc(33, "x"), + }); + + proxy.beginRun("transform-too-large"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{}", + }); + expect(response.status).toBe(413); + expect(await response.json()).toEqual({ error: "request body too large" }); + expect(hits).toBe(0); + expect(proxy.endRun()[0]).toMatchObject({ + visible_tool_count: 0, + status_code: 413, + outcome: "request-rejected", + error_reason: "request-body-too-large", + }); + }); + + it("enforces the frozen temperature after transformation", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requiredTemperature: 0, + requestTransform: ({ body }) => { + const payload = JSON.parse(body.toString("utf8")) as Record; + payload.temperature = 0.5; + return Buffer.from(JSON.stringify(payload), "utf8"); + }, + }); + + proxy.beginRun("transform-temperature"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[],"tools":[]}', + }); + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ error: "performance test temperature mismatch" }); + expect(hits).toBe(0); + expect(proxy.endRun()[0]).toMatchObject({ + status_code: 400, + outcome: "request-rejected", + error_reason: "proxy-failure", + }); + }); + it("streams SSE bytes unchanged and records monotonic first-byte timing", async () => { const chunks = [ 'data: {"choices":[{"delta":{"content":"A"}}]}\n\n', @@ -472,6 +704,46 @@ describe("tool-disclosure recording proxy", () => { ); }); + it("aborts a stalled request transform before an upstream call begins", async () => { + let upstreamHits = 0; + let transformAborted = false; + const upstream = await startUpstream((_request, response) => { + upstreamHits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTimeoutMs: 30, + requestTransform: ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => { + transformAborted = true; + reject(new DOMException("aborted", "AbortError")); + }, + { once: true }, + ); + }), + }); + + proxy.beginRun("transform-timeout"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[],"tools":[]}', + }); + + expect(response.status).toBe(408); + expect(await response.json()).toEqual({ error: "request timeout" }); + expect(transformAborted).toBe(true); + expect(upstreamHits).toBe(0); + expect(proxy.endRun()[0]).toMatchObject({ + status_code: 408, + outcome: "request-rejected", + error_reason: "request-timeout", + }); + }); + it("bounds an SSE response that stalls after its first byte", async () => { const upstream = await startUpstream((_request, response) => { response.writeHead(200, { "content-type": "text/event-stream" }); From 61ab28bb8db660b2243a70829af069a4c6a04b68 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 08:37:26 -0700 Subject: [PATCH 18/31] fix(performance): honor required tool choice Signed-off-by: Aaron Erickson --- .../compositional-tool-routing-transform.ts | 12 ++++-- ...re-compositional-routing-transform.test.ts | 41 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts index e3775fc473..00e24c5ca7 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts @@ -341,9 +341,15 @@ export class CompositionalToolRoutingTransform { cached.evidence.forwarded_tool_count = tools.length; return this.bypass(input.body, "named tool choice conflicts with routed catalog"); } - payload.tools = tools - .filter((tool) => !tool.routable || cached.selectedNames.has(tool.name)) - .map((tool) => tool.definition); + const forwardedTools = tools.filter( + (tool) => !tool.routable || cached.selectedNames.has(tool.name), + ); + if (payload.tool_choice === "required" && forwardedTools.length === 0) { + cached.evidence.transform_bypass = "tool-choice-conflict"; + cached.evidence.forwarded_tool_count = tools.length; + return this.bypass(input.body, "required tool choice has no routed tools"); + } + payload.tools = forwardedTools.map((tool) => tool.definition); return Buffer.from(JSON.stringify(payload), "utf8"); } } diff --git a/test/performance/tool-disclosure-compositional-routing-transform.test.ts b/test/performance/tool-disclosure-compositional-routing-transform.test.ts index b90cd65d8a..16d0574d8e 100644 --- a/test/performance/tool-disclosure-compositional-routing-transform.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-transform.test.ts @@ -404,4 +404,45 @@ describe("independent compositional tool request transform", () => { { transform_bypass: "tool-choice-conflict", forwarded_tool_count: 4 }, ]); }); + + it("preserves a required-tool request when routing would expose no tools", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => [] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const payload = JSON.parse(requestBody().toString("utf8")) as { tools: unknown[] } & Record< + string, + unknown + >; + payload.tools = payload.tools.slice(1); + payload.tool_choice = "required"; + const body = Buffer.from(JSON.stringify(payload), "utf8"); + + expect((await transform.requestTransform(input(body))).equals(body)).toBe(true); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { transform_bypass: "tool-choice-conflict", forwarded_tool_count: 3 }, + ]); + }); + + it("rejects an empty required-tool route in strict mode", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => [] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + requireRouting: true, + }); + const payload = JSON.parse(requestBody().toString("utf8")) as { tools: unknown[] } & Record< + string, + unknown + >; + payload.tools = payload.tools.slice(1); + payload.tool_choice = "required"; + + await expect( + transform.requestTransform(input(Buffer.from(JSON.stringify(payload), "utf8"))), + ).rejects.toThrow( + "required compositional routing bypassed: required tool choice has no routed tools", + ); + }); }); From 77af72726cd4a8608f8056aa5e232ecdca34d520 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 08:43:29 -0700 Subject: [PATCH 19/31] test(performance): keep routing coverage linear Signed-off-by: Aaron Erickson --- .../tool-disclosure-performance-smoke.test.ts | 51 ++++++++++--------- ...ain-deepagents-code-proxy-launcher.test.ts | 6 +-- ...ol-disclosure-compositional-router.test.ts | 11 ++-- ...sclosure-compositional-routing-run.test.ts | 12 +++-- ...re-compositional-routing-transform.test.ts | 19 ++++--- 5 files changed, 54 insertions(+), 45 deletions(-) diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 39246c9835..6dd93dcf9d 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -57,10 +57,11 @@ function gitSha(): string { execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8", }).trim(); - if (!/^[a-f0-9]{40}$/u.test(sha)) { - throw new Error("tool-disclosure performance smoke requires a git SHA"); - } - return sha; + return /^[a-f0-9]{40}$/u.test(sha) + ? sha + : (() => { + throw new Error("tool-disclosure performance smoke requires a git SHA"); + })(); } test.skipIf(!shouldRunLiveE2E())( @@ -87,8 +88,9 @@ test.skipIf(!shouldRunLiveE2E())( embedding: { kind: "portable" }, }); } finally { - if (previousRoutingCredential === undefined) delete process.env[routingCredentialEnv]; - else process.env[routingCredentialEnv] = previousRoutingCredential; + previousRoutingCredential === undefined + ? delete process.env[routingCredentialEnv] + : (process.env[routingCredentialEnv] = previousRoutingCredential); } await artifacts.writeJson("compositional-routing-acceptance.json", routingAcceptance); expect(routingAcceptance.acceptance_passed).toBe(true); @@ -99,10 +101,12 @@ test.skipIf(!shouldRunLiveE2E())( const catalogPrefix = generateCatalogPrefix(catalog, CATALOG_SIZE); const primaryTasks = generatePrimaryTaskSet(catalog); const task = primaryTasks.tasks.find((candidate) => candidate.id === TASK_ID); - if (!task) throw new Error(`frozen task ${TASK_ID} is missing`); - if (task.min_catalog_size > catalogPrefix.size) { - throw new Error(`frozen task ${TASK_ID} does not fit the performance smoke catalog`); - } + expect(task, `frozen task ${TASK_ID} is missing`).toBeDefined(); + const frozenTask = task as NonNullable; + expect( + frozenTask.min_catalog_size, + `frozen task ${TASK_ID} does not fit the performance smoke catalog`, + ).toBeLessThanOrEqual(catalogPrefix.size); await artifacts.writeJson("scenario.json", { id: "tool-disclosure-performance-smoke", @@ -210,7 +214,7 @@ test.skipIf(!shouldRunLiveE2E())( openshellBin: process.env.OPENSHELL_BIN, sandboxName: sandboxName(mode), agent: AGENT, - prompt: task.prompt, + prompt: frozenTask.prompt, sessionId: runId, }); mcp.beginRun(runId); @@ -229,7 +233,7 @@ test.skipIf(!shouldRunLiveE2E())( } const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); - const graded = gradeTaskRun(task, calls, finalOutput); + const graded = gradeTaskRun(frozenTask, calls, finalOutput); const outcome = invocation.timedOut ? "timeout" : invocation.exitCode === 0 @@ -253,10 +257,11 @@ test.skipIf(!shouldRunLiveE2E())( const routedToolNames = new Set( catalogPrefix.tools.map((tool) => `${MCP_SERVER_NAME}_${tool.definition.function.name}`), ); - const expectedRoutedToolName = `${MCP_SERVER_NAME}_${task.expected_calls[0]?.tool_name ?? ""}`; - if (!routedToolNames.has(expectedRoutedToolName)) { - throw new Error("the routed replay target is not present in the reviewed catalog"); - } + const expectedRoutedToolName = `${MCP_SERVER_NAME}_${frozenTask.expected_calls[0]?.tool_name ?? ""}`; + expect( + routedToolNames.has(expectedRoutedToolName), + "the routed replay target is not present in the reviewed catalog", + ).toBe(true); const routedTransform = new CompositionalToolRoutingTransform({ decomposer: createOpenAIChatTaskDecomposer({ baseUrl: hosted.endpointUrl, @@ -318,15 +323,13 @@ test.skipIf(!shouldRunLiveE2E())( }; let routedReplay: Record | undefined; - let routeMutationAttempted = false; try { - routeMutationAttempted = true; await configureInferenceRoute(`${routedProxyAddress.base_url}/v1`, "enable"); const driver = buildAgentDriverCommand({ openshellBin: process.env.OPENSHELL_BIN, sandboxName: sandboxName("direct"), agent: AGENT, - prompt: task.prompt, + prompt: frozenTask.prompt, sessionId: routedRunId, }); mcp.beginRun(routedRunId); @@ -349,7 +352,7 @@ test.skipIf(!shouldRunLiveE2E())( const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); const routeEvidence = await routedTransform.consumeEvidence(routedRunId); const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); - const graded = gradeTaskRun(task, calls, finalOutput); + const graded = gradeTaskRun(frozenTask, calls, finalOutput); const outcome = invocation.timedOut ? "timeout" : invocation.exitCode === 0 @@ -378,9 +381,7 @@ test.skipIf(!shouldRunLiveE2E())( ], }; } finally { - if (routeMutationAttempted) { - await configureInferenceRoute(hosted.endpointUrl, "restore"); - } + await configureInferenceRoute(hosted.endpointUrl, "restore"); } await artifacts.writeJson("compositional-routing-agent-smoke.json", routedReplay); @@ -419,8 +420,8 @@ test.skipIf(!shouldRunLiveE2E())( catalog_seed: catalog.seed, catalog_size: CATALOG_SIZE, catalog_tools_sha256: catalogPrefix.tools_sha256, - task_id: task.id, - task_kind: task.kind, + task_id: frozenTask.id, + task_kind: frozenTask.kind, task_set_sha256: primaryTasks.tasks_sha256, repetitions_per_mode: 1, observed_runs: results.length, diff --git a/test/langchain-deepagents-code-proxy-launcher.test.ts b/test/langchain-deepagents-code-proxy-launcher.test.ts index f7ee1ff1d3..096cdceee4 100644 --- a/test/langchain-deepagents-code-proxy-launcher.test.ts +++ b/test/langchain-deepagents-code-proxy-launcher.test.ts @@ -379,9 +379,9 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { const { scriptPath } = makeStartProxyProbeFixture(tempDir); const modeFile = path.join(tempDir, "trusted-tool-disclosure"); fs.rmSync(modeFile); - if (invalidValue !== null) { - fs.writeFileSync(modeFile, `${invalidValue}\n`, { mode: 0o444 }); - } + invalidValue === null + ? undefined + : fs.writeFileSync(modeFile, `${invalidValue}\n`, { mode: 0o444 }); const env = { NEMOCLAW_TOOL_DISCLOSURE: "direct" }; const launcherResult = runLauncher(launcherPath, ["-n", "PONG"], env); diff --git a/test/performance/tool-disclosure-compositional-router.test.ts b/test/performance/tool-disclosure-compositional-router.test.ts index 2823e1791d..3a2c961bf7 100644 --- a/test/performance/tool-disclosure-compositional-router.test.ts +++ b/test/performance/tool-disclosure-compositional-router.test.ts @@ -46,10 +46,13 @@ class KeywordEmbedder implements TextEmbedder { this.calls.push([...texts]); return texts.map((text) => { const normalized = text.toLowerCase(); - if (/calendar|appointment|event/u.test(normalized)) return [3, 0, 0, 0]; - if (/email|notify|message/u.test(normalized)) return [0, 4, 0, 0]; - if (/weather|forecast/u.test(normalized)) return [0, 0, 5, 0]; - return [0, 0, 0, 2]; + return /calendar|appointment|event/u.test(normalized) + ? [3, 0, 0, 0] + : /email|notify|message/u.test(normalized) + ? [0, 4, 0, 0] + : /weather|forecast/u.test(normalized) + ? [0, 0, 5, 0] + : [0, 0, 0, 2]; }); } } diff --git a/test/performance/tool-disclosure-compositional-routing-run.test.ts b/test/performance/tool-disclosure-compositional-routing-run.test.ts index a918105235..c0d376ff54 100644 --- a/test/performance/tool-disclosure-compositional-routing-run.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-run.test.ts @@ -25,14 +25,15 @@ describe("compositional routing acceptance runner", () => { const fixture = COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.find((candidate) => user.includes(candidate.prompt), ); - if (!fixture) return new Response("missing fixture", { status: 400 }); + expect(fixture).toBeDefined(); + const matchedFixture = fixture as NonNullable; return new Response( JSON.stringify({ choices: [ { message: { content: JSON.stringify( - fixture.expected_steps.map((expected) => expected.capability), + matchedFixture.expected_steps.map((expected) => expected.capability), ), }, }, @@ -116,11 +117,12 @@ describe("compositional routing acceptance runner", () => { const fixture = COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.find((candidate) => user.includes(candidate.prompt), ); - if (!fixture) return new Response("missing fixture", { status: 400 }); + expect(fixture).toBeDefined(); + const matchedFixture = fixture as NonNullable; const content = - fixture.id === "route-no-tool-01" + matchedFixture.id === "route-no-tool-01" ? "not-json" - : JSON.stringify(fixture.expected_steps.map((expected) => expected.capability)); + : JSON.stringify(matchedFixture.expected_steps.map((expected) => expected.capability)); return new Response(JSON.stringify({ choices: [{ message: { content } }] }), { status: 200, headers: { "content-type": "application/json" }, diff --git a/test/performance/tool-disclosure-compositional-routing-transform.test.ts b/test/performance/tool-disclosure-compositional-routing-transform.test.ts index 16d0574d8e..fd52af982f 100644 --- a/test/performance/tool-disclosure-compositional-routing-transform.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-transform.test.ts @@ -15,9 +15,11 @@ import { createToolDisclosureRecordingProxy } from "../../scripts/performance/to function vectorFor(text: string): number[] { const normalized = text.toLowerCase(); - if (normalized.includes("calendar")) return [1, 0, 0]; - if (normalized.includes("email")) return [0, 1, 0]; - return [0, 0, 1]; + return normalized.includes("calendar") + ? [1, 0, 0] + : normalized.includes("email") + ? [0, 1, 0] + : [0, 0, 1]; } const embedder: TextEmbedder = { @@ -318,11 +320,12 @@ describe("independent compositional tool request transform", () => { decomposer: { decompose: async (request) => { decompositionSignals.push(request.signal); - if (request.pass === "refined") return ["list calendar events"]; - markInitialStarted?.(); - return new Promise((resolve) => { - releaseInitial = resolve; - }); + return request.pass === "refined" + ? ["list calendar events"] + : (markInitialStarted?.(), + new Promise((resolve) => { + releaseInitial = resolve; + })); }, }, embedder: { From 559f268e9e17bd4798583e360879f718d293de47 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 08:48:34 -0700 Subject: [PATCH 20/31] fix(performance): constrain decomposition output Signed-off-by: Aaron Erickson --- ...ssive-tool-disclosure-performance-test.mdx | 7 +++- .../compositional-tool-routing-adapters.ts | 32 +++++++++++++++++-- .../compositional-tool-routing-run.ts | 15 +++++++-- .../tool-disclosure-performance-smoke.test.ts | 4 +++ ...ure-compositional-routing-adapters.test.ts | 13 +++++++- ...sclosure-compositional-routing-run.test.ts | 14 +++++--- 6 files changed, 74 insertions(+), 11 deletions(-) diff --git a/docs/inference/progressive-tool-disclosure-performance-test.mdx b/docs/inference/progressive-tool-disclosure-performance-test.mdx index 85efe97451..06de9d9f3e 100644 --- a/docs/inference/progressive-tool-disclosure-performance-test.mdx +++ b/docs/inference/progressive-tool-disclosure-performance-test.mdx @@ -61,7 +61,9 @@ Run the same route-only acceptance corpus with a real OpenAI-compatible decompos "model": "recorded-decomposer-model", "revision": "immutable-model-revision", "api_key_env": "MODEL_API_KEY", - "allow_remote": true + "allow_remote": true, + "reasoning_control": "enable_thinking_false", + "json_object_response": true }, "embedding": { "kind": "portable", @@ -79,6 +81,9 @@ npm run performance:tool-disclosure -- route-acceptance \ ``` The command writes `compositional-routing-acceptance.json` and `SHA256SUMS`, exits nonzero when a strict gate fails, and never writes endpoint URLs, credentials, prompts, or decomposed subtask text. +Set `reasoning_control` only when the endpoint supports the matching chat-template extension: `enable_thinking_false` sends `chat_template_kwargs.enable_thinking=false`, while `thinking_false` sends `chat_template_kwargs.thinking=false`. +Set `json_object_response` only when the endpoint supports OpenAI-compatible JSON-object response mode. +Both choices are recorded in the artifact; omit them to use endpoint defaults and prompt-only JSON formatting. The portable embedding mode is a CPU smoke path and is always claim-ineligible. For a semantic retrieval run, set `embedding.kind` to `openai` and provide its `base_url`, `model`, immutable `revision`, optional `api_key_env`, and remote opt-in. Remote opt-in sends the acceptance prompts and synthetic tool metadata to the configured HTTPS services; HTTP is accepted only on loopback. diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts index bc0a4ccad5..3a5dbbc3aa 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts @@ -37,6 +37,10 @@ interface OpenAIAdapterOptions { export interface OpenAIChatDecomposerOptions extends OpenAIAdapterOptions { maxOutputTokens?: number; + /** Endpoint-specific chat-template switch for concise output. */ + reasoningControl?: "enable_thinking_false" | "thinking_false"; + /** Request the OpenAI-compatible JSON-object response mode. */ + jsonObjectResponse?: boolean; } export interface OpenAITextEmbedderOptions extends OpenAIAdapterOptions { @@ -152,9 +156,12 @@ function decompositionMessages( query: string, pass: DecompositionPass, hints: readonly string[], + jsonObjectResponse: boolean, ): Array<{ role: "system" | "user"; content: string }> { const system = [ - "Return only a JSON array of strings.", + jsonObjectResponse + ? 'Return only a JSON object with one field named "subtasks", whose value is an array of strings.' + : "Return only a JSON array of strings.", "Split the request into a short ordered list of concrete actions.", "Each action must be solvable with one external capability.", "Return an empty array when the request needs no external capability.", @@ -205,6 +212,16 @@ function parseDecompositionResponse(value: unknown): unknown { export function createOpenAIChatTaskDecomposer( options: OpenAIChatDecomposerOptions, ): TaskDecomposer { + if ( + options.reasoningControl !== undefined && + options.reasoningControl !== "enable_thinking_false" && + options.reasoningControl !== "thinking_false" + ) { + throw new TypeError("reasoningControl is not supported"); + } + if (options.jsonObjectResponse !== undefined && typeof options.jsonObjectResponse !== "boolean") { + throw new TypeError("jsonObjectResponse must be boolean"); + } const target = endpoint(options.baseUrl, "chat/completions", options.allowRemote === true); const model = boundedText(options.model, "decomposition model"); const timeoutMs = positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS, "timeoutMs"); @@ -223,10 +240,21 @@ export function createOpenAIChatTaskDecomposer( headers: requestHeaders(options.apiKey), body: JSON.stringify({ model, - messages: decompositionMessages(query, request.pass, request.tool_hints), + messages: decompositionMessages( + query, + request.pass, + request.tool_hints, + options.jsonObjectResponse === true, + ), temperature: 0, max_tokens: maxOutputTokens, stream: false, + ...(options.reasoningControl === "enable_thinking_false" + ? { chat_template_kwargs: { enable_thinking: false } } + : options.reasoningControl === "thinking_false" + ? { chat_template_kwargs: { thinking: false } } + : {}), + ...(options.jsonObjectResponse ? { response_format: { type: "json_object" } } : {}), }), signal: boundedSignal(timeoutMs, request.signal), }); diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts index 894de1182c..92e43c9b95 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts @@ -34,6 +34,8 @@ export interface CompositionalRoutingRemoteModelConfig { revision: string; api_key_env?: string; allow_remote?: boolean; + reasoning_control?: "enable_thinking_false" | "thinking_false"; + json_object_response?: boolean; } export interface CompositionalRoutingAcceptanceConfig { @@ -51,6 +53,8 @@ export interface CompositionalRoutingAcceptanceOutput { configuration: { decomposer_model: string; decomposer_revision: string; + decomposer_reasoning_control: "enable_thinking_false" | "thinking_false" | "endpoint-default"; + decomposer_output_mode: "json-object" | "prompt-only"; embedding_kind: "portable" | "openai"; embedding_model: string; embedding_revision: string; @@ -214,9 +218,11 @@ export async function runCompositionalRoutingAcceptance( } const usage: ModelUsageEvent[] = []; const decomposerRevision = publicIdentity(config.decomposer.revision, "decomposer revision"); - const decomposer = createOpenAIChatTaskDecomposer( - remoteOptions(config.decomposer, timeoutMs, usage), - ); + const decomposer = createOpenAIChatTaskDecomposer({ + ...remoteOptions(config.decomposer, timeoutMs, usage), + reasoningControl: config.decomposer.reasoning_control, + jsonObjectResponse: config.decomposer.json_object_response, + }); const embeddingKind = config.embedding.kind; const embeddingModel = embeddingKind === "portable" @@ -265,6 +271,9 @@ export async function runCompositionalRoutingAcceptance( configuration: { decomposer_model: publicIdentity(config.decomposer.model, "decomposer model"), decomposer_revision: decomposerRevision, + decomposer_reasoning_control: config.decomposer.reasoning_control ?? "endpoint-default", + decomposer_output_mode: + config.decomposer.json_object_response === true ? "json-object" : "prompt-only", embedding_kind: embeddingKind, embedding_model: embeddingModel, embedding_revision: embeddingRevision, diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 6dd93dcf9d..a9d52f78cb 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -84,6 +84,8 @@ test.skipIf(!shouldRunLiveE2E())( revision: process.env.NEMOCLAW_MODEL_REVISION?.trim() || "unreported", api_key_env: routingCredentialEnv, allow_remote: true, + reasoning_control: "enable_thinking_false", + json_object_response: true, }, embedding: { kind: "portable" }, }); @@ -268,6 +270,8 @@ test.skipIf(!shouldRunLiveE2E())( model: hosted.model, apiKey: hosted.apiKey, allowRemote: true, + reasoningControl: "enable_thinking_false", + jsonObjectResponse: true, }), embedder: new PortableHashingTextEmbedder(), isRoutableTool: (name) => routedToolNames.has(name), diff --git a/test/performance/tool-disclosure-compositional-routing-adapters.test.ts b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts index 8fa6fb4dd4..02b54513b9 100644 --- a/test/performance/tool-disclosure-compositional-routing-adapters.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts @@ -25,7 +25,7 @@ describe("independent compositional tool model adapters", () => { headers: new Headers(init?.headers), body: JSON.parse(String(init?.body)) as Record, }); - const content = JSON.stringify(responses[calls.length - 1]); + const content = JSON.stringify({ subtasks: responses[calls.length - 1] }); return new Response( JSON.stringify({ choices: [{ message: { content: `\`\`\`json\n${content}\n\`\`\`` } }], @@ -41,6 +41,8 @@ describe("independent compositional tool model adapters", () => { model: "test-model", allowRemote: true, apiKey: "private-api-key", + reasoningControl: "enable_thinking_false", + jsonObjectResponse: true, onUsage: (event) => usage.push(event), }); @@ -67,6 +69,8 @@ describe("independent compositional tool model adapters", () => { temperature: 0, max_tokens: 256, stream: false, + chat_template_kwargs: { enable_thinking: false }, + response_format: { type: "json_object" }, }); expect(JSON.stringify(calls[1].body)).toContain("route_fetch"); expect(usage).toHaveLength(2); @@ -159,5 +163,12 @@ describe("independent compositional tool model adapters", () => { allowRemote: true, }), ).toThrow("must use HTTPS"); + expect(() => + createOpenAIChatTaskDecomposer({ + baseUrl: "http://127.0.0.1:8000/v1", + model: "test-model", + reasoningControl: "invalid" as "thinking_false", + }), + ).toThrow("reasoningControl is not supported"); }); }); diff --git a/test/performance/tool-disclosure-compositional-routing-run.test.ts b/test/performance/tool-disclosure-compositional-routing-run.test.ts index c0d376ff54..a4957df718 100644 --- a/test/performance/tool-disclosure-compositional-routing-run.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-run.test.ts @@ -32,9 +32,9 @@ describe("compositional routing acceptance runner", () => { choices: [ { message: { - content: JSON.stringify( - matchedFixture.expected_steps.map((expected) => expected.capability), - ), + content: JSON.stringify({ + subtasks: matchedFixture.expected_steps.map((expected) => expected.capability), + }), }, }, ], @@ -52,6 +52,8 @@ describe("compositional routing acceptance runner", () => { revision: "immutable-test-revision", api_key_env: "ROUTING_TEST_API_KEY", allow_remote: true, + reasoning_control: "enable_thinking_false", + json_object_response: true, }, embedding: { kind: "portable", dimensions: 1_024 }, }); @@ -62,6 +64,8 @@ describe("compositional routing acceptance runner", () => { configuration: { decomposer_model: "test-decomposer", decomposer_revision: "immutable-test-revision", + decomposer_reasoning_control: "enable_thinking_false", + decomposer_output_mode: "json-object", embedding_kind: "portable", embedding_model: "portable-lexical-hashing", embedding_revision: "builtin-v1", @@ -122,7 +126,9 @@ describe("compositional routing acceptance runner", () => { const content = matchedFixture.id === "route-no-tool-01" ? "not-json" - : JSON.stringify(matchedFixture.expected_steps.map((expected) => expected.capability)); + : JSON.stringify({ + subtasks: matchedFixture.expected_steps.map((expected) => expected.capability), + }); return new Response(JSON.stringify({ choices: [{ message: { content } }] }), { status: 200, headers: { "content-type": "application/json" }, From e203c7a9c4a04e765c842e4e623277f8860f8170 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 09:00:02 -0700 Subject: [PATCH 21/31] fix(performance): bound decomposer retries Signed-off-by: Aaron Erickson --- ...ssive-tool-disclosure-performance-test.mdx | 7 +- .../compositional-tool-routing-adapters.ts | 97 +++++++++++-------- .../compositional-tool-routing-run.ts | 10 ++ .../tool-disclosure-performance-smoke.test.ts | 11 +++ ...ure-compositional-routing-adapters.test.ts | 35 +++++++ ...sclosure-compositional-routing-run.test.ts | 6 +- 6 files changed, 123 insertions(+), 43 deletions(-) diff --git a/docs/inference/progressive-tool-disclosure-performance-test.mdx b/docs/inference/progressive-tool-disclosure-performance-test.mdx index 06de9d9f3e..2c0bd583fb 100644 --- a/docs/inference/progressive-tool-disclosure-performance-test.mdx +++ b/docs/inference/progressive-tool-disclosure-performance-test.mdx @@ -63,12 +63,14 @@ Run the same route-only acceptance corpus with a real OpenAI-compatible decompos "api_key_env": "MODEL_API_KEY", "allow_remote": true, "reasoning_control": "enable_thinking_false", - "json_object_response": true + "json_object_response": true, + "max_attempts": 2 }, "embedding": { "kind": "portable", "dimensions": 1024 - } + }, + "timeout_ms": 180000 } ``` @@ -84,6 +86,7 @@ The command writes `compositional-routing-acceptance.json` and `SHA256SUMS`, exi Set `reasoning_control` only when the endpoint supports the matching chat-template extension: `enable_thinking_false` sends `chat_template_kwargs.enable_thinking=false`, while `thinking_false` sends `chat_template_kwargs.thinking=false`. Set `json_object_response` only when the endpoint supports OpenAI-compatible JSON-object response mode. Both choices are recorded in the artifact; omit them to use endpoint defaults and prompt-only JSON formatting. +`max_attempts` is a bounded per-decomposition retry count, and `timeout_ms` applies to each remote request; both are recorded, and every attempt contributes to reported usage and latency. The portable embedding mode is a CPU smoke path and is always claim-ineligible. For a semantic retrieval run, set `embedding.kind` to `openai` and provide its `base_url`, `model`, immutable `revision`, optional `api_key_env`, and remote opt-in. Remote opt-in sends the acceptance prompts and synthetic tool metadata to the configured HTTPS services; HTTP is accepted only on loopback. diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts index 3a5dbbc3aa..2ade70504d 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts @@ -19,6 +19,8 @@ interface JsonRecord { export interface ModelUsageEvent { operation: "decomposition" | "embedding"; pass?: DecompositionPass; + attempt?: number; + outcome?: "completed" | "failed"; prompt_tokens?: number; completion_tokens?: number; total_tokens?: number; @@ -37,6 +39,7 @@ interface OpenAIAdapterOptions { export interface OpenAIChatDecomposerOptions extends OpenAIAdapterOptions { maxOutputTokens?: number; + maxAttempts?: number; /** Endpoint-specific chat-template switch for concise output. */ reasoningControl?: "enable_thinking_false" | "thinking_false"; /** Request the OpenAI-compatible JSON-object response mode. */ @@ -229,50 +232,61 @@ export function createOpenAIChatTaskDecomposer( options.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS, "maxOutputTokens", ); + const maxAttempts = positiveInteger(options.maxAttempts ?? 1, "maxAttempts"); return { decompose: async (request) => { const query = boundedText(request.query, "decomposition query"); - const startedAt = performance.now(); - let body: unknown; - try { - const response = await fetch(target, { - method: "POST", - headers: requestHeaders(options.apiKey), - body: JSON.stringify({ - model, - messages: decompositionMessages( - query, - request.pass, - request.tool_hints, - options.jsonObjectResponse === true, - ), - temperature: 0, - max_tokens: maxOutputTokens, - stream: false, - ...(options.reasoningControl === "enable_thinking_false" - ? { chat_template_kwargs: { enable_thinking: false } } - : options.reasoningControl === "thinking_false" - ? { chat_template_kwargs: { thinking: false } } - : {}), - ...(options.jsonObjectResponse ? { response_format: { type: "json_object" } } : {}), - }), - signal: boundedSignal(timeoutMs, request.signal), - }); - if (!response.ok) throw new Error("decomposition request failed"); - body = await readBoundedJson(response); - } catch { - throw new Error("decomposition request failed"); - } finally { - // Never include prompts, endpoints, model IDs, keys, or response text. - const usage = isRecord(body) ? usageFrom(body.usage) : {}; - reportUsage(options.onUsage, { - operation: "decomposition", - pass: request.pass, - ...usage, - duration_ms: performance.now() - startedAt, - }); + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const startedAt = performance.now(); + let body: unknown; + let outcome: ModelUsageEvent["outcome"] = "failed"; + try { + const response = await fetch(target, { + method: "POST", + headers: requestHeaders(options.apiKey), + body: JSON.stringify({ + model, + messages: decompositionMessages( + query, + request.pass, + request.tool_hints, + options.jsonObjectResponse === true, + ), + temperature: 0, + max_tokens: maxOutputTokens, + stream: false, + ...(options.reasoningControl === "enable_thinking_false" + ? { chat_template_kwargs: { enable_thinking: false } } + : options.reasoningControl === "thinking_false" + ? { chat_template_kwargs: { thinking: false } } + : {}), + ...(options.jsonObjectResponse ? { response_format: { type: "json_object" } } : {}), + }), + signal: boundedSignal(timeoutMs, request.signal), + }); + if (!response.ok) throw new Error("decomposition request failed"); + body = await readBoundedJson(response); + const parsed = parseDecompositionResponse(body); + outcome = "completed"; + return parsed; + } catch { + if (request.signal?.aborted || attempt === maxAttempts) { + throw new Error("decomposition request failed"); + } + } finally { + // Never include prompts, endpoints, model IDs, keys, or response text. + const usage = isRecord(body) ? usageFrom(body.usage) : {}; + reportUsage(options.onUsage, { + operation: "decomposition", + pass: request.pass, + attempt, + outcome, + ...usage, + duration_ms: performance.now() - startedAt, + }); + } } - return parseDecompositionResponse(body); + throw new Error("decomposition request failed"); }, }; } @@ -316,6 +330,7 @@ export function createOpenAITextEmbedder(options: OpenAITextEmbedderOptions): Te .map((text) => boundedText(text, "embedding input")); const startedAt = performance.now(); let body: unknown; + let outcome: ModelUsageEvent["outcome"] = "failed"; try { const response = await fetch(target, { method: "POST", @@ -326,12 +341,14 @@ export function createOpenAITextEmbedder(options: OpenAITextEmbedderOptions): Te if (!response.ok) throw new Error("embedding request failed"); body = await readBoundedJson(response); output.push(...parseEmbeddingResponse(body, input.length)); + outcome = "completed"; } catch { throw new Error("embedding request failed"); } finally { const usage = isRecord(body) ? usageFrom(body.usage) : {}; reportUsage(options.onUsage, { operation: "embedding", + outcome, ...usage, duration_ms: performance.now() - startedAt, }); diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts index 92e43c9b95..be89a08baa 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts @@ -36,6 +36,7 @@ export interface CompositionalRoutingRemoteModelConfig { allow_remote?: boolean; reasoning_control?: "enable_thinking_false" | "thinking_false"; json_object_response?: boolean; + max_attempts?: number; } export interface CompositionalRoutingAcceptanceConfig { @@ -55,6 +56,8 @@ export interface CompositionalRoutingAcceptanceOutput { decomposer_revision: string; decomposer_reasoning_control: "enable_thinking_false" | "thinking_false" | "endpoint-default"; decomposer_output_mode: "json-object" | "prompt-only"; + decomposer_max_attempts: number; + request_timeout_ms: number; embedding_kind: "portable" | "openai"; embedding_model: string; embedding_revision: string; @@ -72,6 +75,7 @@ export interface CompositionalRoutingAcceptanceOutput { usage: { decomposition: { requests: number; + failed_requests: number; prompt_tokens?: number; completion_tokens?: number; total_tokens?: number; @@ -79,6 +83,7 @@ export interface CompositionalRoutingAcceptanceOutput { }; embedding: { requests: number; + failed_requests: number; prompt_tokens?: number; completion_tokens?: number; total_tokens?: number; @@ -201,6 +206,7 @@ function operationUsage( const totalTokens = sumUsage(selected, "total_tokens"); return { requests: selected.length, + failed_requests: selected.filter((event) => event.outcome === "failed").length, ...(promptTokens === undefined ? {} : { prompt_tokens: promptTokens }), ...(completionTokens === undefined ? {} : { completion_tokens: completionTokens }), ...(totalTokens === undefined ? {} : { total_tokens: totalTokens }), @@ -218,10 +224,12 @@ export async function runCompositionalRoutingAcceptance( } const usage: ModelUsageEvent[] = []; const decomposerRevision = publicIdentity(config.decomposer.revision, "decomposer revision"); + const decomposerMaxAttempts = config.decomposer.max_attempts ?? 1; const decomposer = createOpenAIChatTaskDecomposer({ ...remoteOptions(config.decomposer, timeoutMs, usage), reasoningControl: config.decomposer.reasoning_control, jsonObjectResponse: config.decomposer.json_object_response, + maxAttempts: decomposerMaxAttempts, }); const embeddingKind = config.embedding.kind; const embeddingModel = @@ -274,6 +282,8 @@ export async function runCompositionalRoutingAcceptance( decomposer_reasoning_control: config.decomposer.reasoning_control ?? "endpoint-default", decomposer_output_mode: config.decomposer.json_object_response === true ? "json-object" : "prompt-only", + decomposer_max_attempts: decomposerMaxAttempts, + request_timeout_ms: timeoutMs, embedding_kind: embeddingKind, embedding_model: embeddingModel, embedding_revision: embeddingRevision, diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index a9d52f78cb..b1995bf39c 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -86,8 +86,10 @@ test.skipIf(!shouldRunLiveE2E())( allow_remote: true, reasoning_control: "enable_thinking_false", json_object_response: true, + max_attempts: 2, }, embedding: { kind: "portable" }, + timeout_ms: 180_000, }); } finally { previousRoutingCredential === undefined @@ -272,6 +274,8 @@ test.skipIf(!shouldRunLiveE2E())( allowRemote: true, reasoningControl: "enable_thinking_false", jsonObjectResponse: true, + maxAttempts: 2, + timeoutMs: 180_000, }), embedder: new PortableHashingTextEmbedder(), isRoutableTool: (name) => routedToolNames.has(name), @@ -376,6 +380,13 @@ test.skipIf(!shouldRunLiveE2E())( synthetic_call_count: calls.length, correctness: graded.correctness, expected_routed_tool_name: expectedRoutedToolName, + routing_configuration: { + reasoning_control: "enable_thinking_false", + output_mode: "json-object", + max_attempts: 2, + timeout_ms: 180_000, + embedding: "portable-lexical-hashing", + }, route_evidence: routeEvidence, recorder_events: recorderEvents, limitations: [ diff --git a/test/performance/tool-disclosure-compositional-routing-adapters.test.ts b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts index 02b54513b9..df6d763f06 100644 --- a/test/performance/tool-disclosure-compositional-routing-adapters.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts @@ -100,6 +100,41 @@ describe("independent compositional tool model adapters", () => { ).rejects.toThrow("decomposition request failed"); }); + it("retries decomposition within the configured bound and records every attempt", async () => { + let attempts = 0; + vi.stubGlobal( + "fetch", + vi.fn(async () => { + attempts += 1; + return attempts === 1 + ? new Response("temporary failure", { status: 503 }) + : new Response( + JSON.stringify({ + choices: [{ message: { content: '{"subtasks":["retry succeeded"]}' } }], + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }), + ); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 2, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).resolves.toEqual(["retry succeeded"]); + expect(attempts).toBe(2); + expect(usage).toMatchObject([ + { attempt: 1, outcome: "failed" }, + { attempt: 2, outcome: "completed" }, + ]); + }); + it("batches embeddings and restores provider results to input order", async () => { const observedInputs: string[][] = []; vi.stubGlobal( diff --git a/test/performance/tool-disclosure-compositional-routing-run.test.ts b/test/performance/tool-disclosure-compositional-routing-run.test.ts index a4957df718..31170a4331 100644 --- a/test/performance/tool-disclosure-compositional-routing-run.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-run.test.ts @@ -54,6 +54,7 @@ describe("compositional routing acceptance runner", () => { allow_remote: true, reasoning_control: "enable_thinking_false", json_object_response: true, + max_attempts: 2, }, embedding: { kind: "portable", dimensions: 1_024 }, }); @@ -66,6 +67,8 @@ describe("compositional routing acceptance runner", () => { decomposer_revision: "immutable-test-revision", decomposer_reasoning_control: "enable_thinking_false", decomposer_output_mode: "json-object", + decomposer_max_attempts: 2, + request_timeout_ms: 120_000, embedding_kind: "portable", embedding_model: "portable-lexical-hashing", embedding_revision: "builtin-v1", @@ -77,11 +80,12 @@ describe("compositional routing acceptance runner", () => { usage: { decomposition: { requests: 15, + failed_requests: 0, prompt_tokens: 150, completion_tokens: 30, total_tokens: 180, }, - embedding: { requests: 0 }, + embedding: { requests: 0, failed_requests: 0 }, }, acceptance_passed: true, }); From 18e3c8e017ea309999141581c2e8fe6d16c78e2f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 09:17:29 -0700 Subject: [PATCH 22/31] fix(performance): harden compositional routing acceptance Signed-off-by: Aaron Erickson --- ...ssive-tool-disclosure-performance-test.mdx | 10 +- .../compositional-tool-router.ts | 1 + .../compositional-tool-routing-adapters.ts | 75 +++++++- .../compositional-tool-routing-run.ts | 131 +++++++++++-- .../compositional-tool-routing-transform.ts | 66 +++++-- .../tool-disclosure-performance-smoke.test.ts | 24 ++- ...ure-compositional-routing-adapters.test.ts | 175 ++++++++++++++++++ ...sclosure-compositional-routing-run.test.ts | 104 +++++++++++ ...re-compositional-routing-transform.test.ts | 59 +++++- 9 files changed, 597 insertions(+), 48 deletions(-) diff --git a/docs/inference/progressive-tool-disclosure-performance-test.mdx b/docs/inference/progressive-tool-disclosure-performance-test.mdx index 2c0bd583fb..49eb65e393 100644 --- a/docs/inference/progressive-tool-disclosure-performance-test.mdx +++ b/docs/inference/progressive-tool-disclosure-performance-test.mdx @@ -70,7 +70,8 @@ Run the same route-only acceptance corpus with a real OpenAI-compatible decompos "kind": "portable", "dimensions": 1024 }, - "timeout_ms": 180000 + "timeout_ms": 120000, + "run_timeout_ms": 900000 } ``` @@ -86,7 +87,12 @@ The command writes `compositional-routing-acceptance.json` and `SHA256SUMS`, exi Set `reasoning_control` only when the endpoint supports the matching chat-template extension: `enable_thinking_false` sends `chat_template_kwargs.enable_thinking=false`, while `thinking_false` sends `chat_template_kwargs.thinking=false`. Set `json_object_response` only when the endpoint supports OpenAI-compatible JSON-object response mode. Both choices are recorded in the artifact; omit them to use endpoint defaults and prompt-only JSON formatting. -`max_attempts` is a bounded per-decomposition retry count, and `timeout_ms` applies to each remote request; both are recorded, and every attempt contributes to reported usage and latency. +`max_attempts` is the per-decomposition attempt count and must be at most `3`. +`timeout_ms` applies to each remote request and must be at most `300000`; the product of `timeout_ms` and `max_attempts` must not exceed `600000`. +`run_timeout_ms` is the overall route-only deadline across index construction and every corpus case and must be at most `2700000`. +The overall deadline aborts outstanding remote work, exits nonzero, and is recorded with the per-request and retry limits. +Timed-out runs record an explicit execution status, completed-case count, and `run-deadline-exceeded` case evidence instead of attributing unattempted work to a model or embedding failure. +Every attempt contributes to the request and duration counts, and any provider token usage returned for an attempt is aggregated. The portable embedding mode is a CPU smoke path and is always claim-ineligible. For a semantic retrieval run, set `embedding.kind` to `openai` and provide its `base_url`, `model`, immutable `revision`, optional `api_key_env`, and remote opt-in. Remote opt-in sends the acceptance prompts and synthetic tool metadata to the configured HTTPS services; HTTP is accepted only on loopback. diff --git a/scripts/performance/tool-disclosure/compositional-tool-router.ts b/scripts/performance/tool-disclosure/compositional-tool-router.ts index 84cf274330..4971b55f90 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-router.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-router.ts @@ -71,6 +71,7 @@ export type RoutingFallbackReason = | "refinement-failed" | "refinement-malformed" | "refinement-no-tool-disagreement" + | "run-deadline-exceeded" | "selection-limit-exceeded"; export interface CompositionalRoutingTimings { diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts index 2ade70504d..5f9ef36fa2 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts @@ -3,7 +3,13 @@ import { performance } from "node:perf_hooks"; -import type { DecompositionPass, TaskDecomposer, TextEmbedder } from "./compositional-tool-router"; +import { + COMPOSITIONAL_ROUTER_MAX_SUBTASK_CHARS, + COMPOSITIONAL_ROUTER_MAX_SUBTASKS, + type DecompositionPass, + type TaskDecomposer, + type TextEmbedder, +} from "./compositional-tool-router"; const DEFAULT_TIMEOUT_MS = 120_000; const DEFAULT_MAX_OUTPUT_TOKENS = 256; @@ -11,6 +17,9 @@ const DEFAULT_EMBEDDING_BATCH_SIZE = 128; const DEFAULT_HASH_DIMENSIONS = 1_024; const MAX_TEXT_CHARS = 32_768; const MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +export const MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS = 3; +export const MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS = 300_000; +export const MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS = 600_000; interface JsonRecord { [key: string]: unknown; @@ -61,6 +70,18 @@ function positiveInteger(value: number, label: string): number { return value; } +function boundedPositiveInteger(value: number, label: string, maximum: number): number { + const validated = positiveInteger(value, label); + if (validated > maximum) throw new RangeError(`${label} must not exceed ${maximum}`); + return validated; +} + +class ModelHttpStatusError extends Error { + constructor(readonly status: number) { + super("model request failed"); + } +} + function endpoint( baseUrl: string, leaf: "chat/completions" | "embeddings", @@ -193,7 +214,7 @@ function jsonPayloadText(value: string): string { return trimmed.slice(firstNewline + 1, lastFence).trim(); } -function parseDecompositionResponse(value: unknown): unknown { +function parseDecompositionResponse(value: unknown): string[] { if (!isRecord(value) || !Array.isArray(value.choices)) { throw new Error("decomposition response has an unsupported shape"); } @@ -208,7 +229,26 @@ function parseDecompositionResponse(value: unknown): unknown { } catch { throw new Error("decomposition response contains invalid JSON"); } - return isRecord(parsed) && Array.isArray(parsed.subtasks) ? parsed.subtasks : parsed; + const subtasks = isRecord(parsed) && Array.isArray(parsed.subtasks) ? parsed.subtasks : parsed; + if (!Array.isArray(subtasks) || subtasks.length > COMPOSITIONAL_ROUTER_MAX_SUBTASKS) { + throw new Error("decomposition response has an invalid subtask array"); + } + return subtasks.map((item) => { + if (typeof item !== "string") { + throw new Error("decomposition response has a non-string subtask"); + } + const normalized = item.trim(); + if (!normalized || normalized.length > COMPOSITIONAL_ROUTER_MAX_SUBTASK_CHARS) { + throw new Error("decomposition response has an invalid subtask"); + } + return normalized; + }); +} + +function retryableDecompositionError(error: unknown, parent: AbortSignal | undefined): boolean { + if (parent?.aborted) return false; + if (!(error instanceof ModelHttpStatusError)) return true; + return error.status === 408 || error.status === 429 || error.status >= 500; } /** Build the two-pass decomposer using any OpenAI-compatible chat endpoint. */ @@ -227,12 +267,25 @@ export function createOpenAIChatTaskDecomposer( } const target = endpoint(options.baseUrl, "chat/completions", options.allowRemote === true); const model = boundedText(options.model, "decomposition model"); - const timeoutMs = positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS, "timeoutMs"); + const timeoutMs = boundedPositiveInteger( + options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + "timeoutMs", + MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS, + ); const maxOutputTokens = positiveInteger( options.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS, "maxOutputTokens", ); - const maxAttempts = positiveInteger(options.maxAttempts ?? 1, "maxAttempts"); + const maxAttempts = boundedPositiveInteger( + options.maxAttempts ?? 1, + "maxAttempts", + MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS, + ); + if (timeoutMs * maxAttempts > MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS) { + throw new RangeError( + `timeoutMs * maxAttempts must not exceed ${MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS}`, + ); + } return { decompose: async (request) => { const query = boundedText(request.query, "decomposition query"); @@ -264,13 +317,13 @@ export function createOpenAIChatTaskDecomposer( }), signal: boundedSignal(timeoutMs, request.signal), }); - if (!response.ok) throw new Error("decomposition request failed"); + if (!response.ok) throw new ModelHttpStatusError(response.status); body = await readBoundedJson(response); const parsed = parseDecompositionResponse(body); outcome = "completed"; return parsed; - } catch { - if (request.signal?.aborted || attempt === maxAttempts) { + } catch (error) { + if (!retryableDecompositionError(error, request.signal) || attempt === maxAttempts) { throw new Error("decomposition request failed"); } } finally { @@ -319,7 +372,11 @@ function parseEmbeddingResponse(value: unknown, expected: number): number[][] { export function createOpenAITextEmbedder(options: OpenAITextEmbedderOptions): TextEmbedder { const target = endpoint(options.baseUrl, "embeddings", options.allowRemote === true); const model = boundedText(options.model, "embedding model"); - const timeoutMs = positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS, "timeoutMs"); + const timeoutMs = boundedPositiveInteger( + options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + "timeoutMs", + MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS, + ); const batchSize = positiveInteger(options.batchSize ?? DEFAULT_EMBEDDING_BATCH_SIZE, "batchSize"); return { embed: async (texts, signal) => { diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts index be89a08baa..7f1f8a6478 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts @@ -22,11 +22,14 @@ import { import { createOpenAIChatTaskDecomposer, createOpenAITextEmbedder, + MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS, type ModelUsageEvent, PortableHashingTextEmbedder, } from "./compositional-tool-routing-adapters"; const SAFE_ENV_NAME = /^[A-Z][A-Z0-9_]{0,127}$/u; +const DEFAULT_RUN_TIMEOUT_MS = 15 * 60_000; +const MAX_RUN_TIMEOUT_MS = 45 * 60_000; export interface CompositionalRoutingRemoteModelConfig { base_url: string; @@ -45,6 +48,7 @@ export interface CompositionalRoutingAcceptanceConfig { | { kind: "portable"; dimensions?: number } | ({ kind: "openai" } & CompositionalRoutingRemoteModelConfig); timeout_ms?: number; + run_timeout_ms?: number; } export interface CompositionalRoutingAcceptanceOutput { @@ -58,6 +62,7 @@ export interface CompositionalRoutingAcceptanceOutput { decomposer_output_mode: "json-object" | "prompt-only"; decomposer_max_attempts: number; request_timeout_ms: number; + run_timeout_ms: number; embedding_kind: "portable" | "openai"; embedding_model: string; embedding_revision: string; @@ -92,11 +97,17 @@ export interface CompositionalRoutingAcceptanceOutput { index_build_time_ms: number; end_to_end_time_ms: number; }; + execution: { + status: "completed" | "timed-out"; + completed_case_count: number; + total_case_count: number; + }; comparison: CompositionalRoutingComparison; cases: readonly { case_id: string; expected_step_count: number; - shared_initial_decomposition: true; + evaluation_status: "completed" | "run-deadline-exceeded"; + shared_initial_decomposition: boolean; initial: { disposition: CompositionalRoutingResult["disposition"]; forwarded_tool_names: readonly string[]; @@ -186,6 +197,35 @@ function observation(result: CompositionalRoutingResult) { }; } +function runDeadlineResult(): CompositionalRoutingResult { + return { + disposition: "passthrough", + selected_tool_names: [], + evidence: { + initial_subtask_count: 0, + refined_subtask_count: 0, + decomposition_passes: 0, + hint_count: 0, + hint_tool_names: [], + initial_candidate_counts: [], + initial_candidate_tool_names: [], + final_candidate_counts: [], + final_candidate_tool_names: [], + selected_tool_count: 0, + selected_tool_names: [], + fallback: "run-deadline-exceeded", + timings: { + initial_decomposition_ms: 0, + catalog_embedding_ms: 0, + initial_retrieval_ms: 0, + refinement_ms: 0, + final_retrieval_ms: 0, + total_ms: 0, + }, + }, + }; +} + function sumUsage( events: readonly ModelUsageEvent[], key: "prompt_tokens" | "completion_tokens" | "total_tokens", @@ -219,8 +259,24 @@ export async function runCompositionalRoutingAcceptance( config: CompositionalRoutingAcceptanceConfig, ): Promise { const timeoutMs = config.timeout_ms ?? 120_000; - if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { - throw new TypeError("timeout_ms must be a positive safe integer"); + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs <= 0 || + timeoutMs > MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS + ) { + throw new TypeError( + `timeout_ms must be a positive safe integer no greater than ${MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS}`, + ); + } + const runTimeoutMs = config.run_timeout_ms ?? DEFAULT_RUN_TIMEOUT_MS; + if ( + !Number.isSafeInteger(runTimeoutMs) || + runTimeoutMs <= 0 || + runTimeoutMs > MAX_RUN_TIMEOUT_MS + ) { + throw new TypeError( + `run_timeout_ms must be a positive safe integer no greater than ${MAX_RUN_TIMEOUT_MS}`, + ); } const usage: ModelUsageEvent[] = []; const decomposerRevision = publicIdentity(config.decomposer.revision, "decomposer revision"); @@ -246,32 +302,77 @@ export async function runCompositionalRoutingAcceptance( : createOpenAITextEmbedder(remoteOptions(config.embedding, timeoutMs, usage)); const startedAt = performance.now(); + const deadlineAt = startedAt + runTimeoutMs; + const runSignal = AbortSignal.timeout(runTimeoutMs); + const deadlineReached = () => runSignal.aborted || performance.now() >= deadlineAt; const indexStartedAt = performance.now(); - const preparedIndex = await buildDenseToolIndex(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, embedder); + let preparedIndex: Awaited> | null = null; + let indexError: unknown; + let indexFailed = false; + try { + preparedIndex = await buildDenseToolIndex( + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + embedder, + runSignal, + ); + } catch (error) { + indexFailed = true; + indexError = error; + } const indexBuildTimeMs = performance.now() - indexStartedAt; + let runTimedOut = deadlineReached(); + if (indexFailed && !runTimedOut) throw indexError; const initialPredictions: CompositionalRoutingCasePrediction[] = []; const refinedPredictions: CompositionalRoutingCasePrediction[] = []; const cases: CompositionalRoutingAcceptanceOutput["cases"][number][] = []; + let completedCaseCount = 0; for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { - const pair = await routeCompositionalToolsPaired( - fixture.prompt, - COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, - { decomposer, embedder, preparedIndex }, - ); + runTimedOut ||= deadlineReached(); + const attemptedPair = + runTimedOut || preparedIndex === null + ? null + : await routeCompositionalToolsPaired( + fixture.prompt, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + { decomposer, embedder, preparedIndex, signal: runSignal }, + ); + runTimedOut ||= deadlineReached(); + const pair = + runTimedOut || attemptedPair === null + ? { + initial: runDeadlineResult(), + refined: runDeadlineResult(), + shared_initial_decomposition: false as const, + } + : attemptedPair; + if (!runTimedOut) completedCaseCount += 1; initialPredictions.push(prediction(fixture.id, pair.initial)); refinedPredictions.push(prediction(fixture.id, pair.refined)); cases.push({ case_id: fixture.id, expected_step_count: fixture.expected_steps.length, - shared_initial_decomposition: true, + evaluation_status: runTimedOut ? "run-deadline-exceeded" : "completed", + shared_initial_decomposition: pair.shared_initial_decomposition, initial: observation(pair.initial), refined: observation(pair.refined), }); } - const comparison = compareCompositionalRoutingVariants( + runTimedOut ||= deadlineReached(); + const evaluatedComparison = compareCompositionalRoutingVariants( { variant: "initial", cases: initialPredictions }, { variant: "refined", cases: refinedPredictions }, ); + runTimedOut ||= deadlineReached(); + const comparison: CompositionalRoutingComparison = runTimedOut + ? { + ...evaluatedComparison, + passed: false, + reasons: [ + ...evaluatedComparison.reasons, + `run deadline exceeded after ${completedCaseCount} of ${COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.length} cases`, + ], + } + : evaluatedComparison; return { schema_version: "nemoclaw.compositional_tool_routing_acceptance.v1", generated_at: new Date().toISOString(), @@ -284,6 +385,7 @@ export async function runCompositionalRoutingAcceptance( config.decomposer.json_object_response === true ? "json-object" : "prompt-only", decomposer_max_attempts: decomposerMaxAttempts, request_timeout_ms: timeoutMs, + run_timeout_ms: runTimeoutMs, embedding_kind: embeddingKind, embedding_model: embeddingModel, embedding_revision: embeddingRevision, @@ -311,9 +413,14 @@ export async function runCompositionalRoutingAcceptance( index_build_time_ms: indexBuildTimeMs, end_to_end_time_ms: performance.now() - startedAt, }, + execution: { + status: runTimedOut ? "timed-out" : "completed", + completed_case_count: completedCaseCount, + total_case_count: COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.length, + }, comparison, cases, - acceptance_passed: comparison.passed, + acceptance_passed: !runTimedOut && comparison.passed, limitations: [ "This route-only acceptance run does not execute an agent or a tool.", "The eight-case corpus is software acceptance coverage, not a universal quality result.", diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts index 00e24c5ca7..91679b3441 100644 --- a/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts @@ -35,6 +35,13 @@ interface CachedRoute { evidence: CompositionalTransformEvidence; } +interface CachedRouteWork { + promise: Promise; + controller: AbortController; + waiterCount: number; + settled: boolean; +} + export interface CompositionalTransformEvidence { run_id: string; source_tool_count: number; @@ -168,7 +175,7 @@ export class CompositionalToolRoutingTransform { readonly requestTransform: RecordingProxyRequestTransform; private readonly isRoutableTool: (name: string) => boolean; - private readonly routes = new Map>(); + private readonly routes = new Map(); private readonly routeKeysByRun = new Map>(); private readonly indexes = new Map>(); @@ -186,11 +193,11 @@ export class CompositionalToolRoutingTransform { this.routeKeysByRun.delete(runId); const evidence: CompositionalTransformEvidence[] = []; for (const key of keys) { - const pending = this.routes.get(key); + const work = this.routes.get(key); this.routes.delete(key); - if (!pending) continue; + if (!work) continue; try { - evidence.push(cloneEvidence((await pending).evidence)); + evidence.push(cloneEvidence((await work.promise).evidence)); } catch { // Aborted/failed routes do not retain partial evidence. } @@ -205,6 +212,13 @@ export class CompositionalToolRoutingTransform { return body; } + private evictRoute(runId: string, routeKey: string, work: CachedRouteWork): void { + if (this.routes.get(routeKey) === work) this.routes.delete(routeKey); + const runKeys = this.routeKeysByRun.get(runId); + runKeys?.delete(routeKey); + if (runKeys?.size === 0) this.routeKeysByRun.delete(runId); + } + private preparedIndex( fingerprint: string, catalog: readonly ToolRoutingCatalogEntry[], @@ -270,22 +284,25 @@ export class CompositionalToolRoutingTransform { const query = extractChatQuery(payload); if (!query) return this.bypass(input.body, "request has no user query"); const routeKey = `${runId}:${queryFingerprint(query)}:${fingerprint}`; - let pending = this.routes.get(routeKey); - const cacheHit = pending !== undefined; + let work = this.routes.get(routeKey); + const cacheHit = work !== undefined; - if (!pending) { + if (!work) { const catalog: ToolRoutingCatalogEntry[] = routable.map((tool) => ({ name: tool.name, description: tool.description, definition: tool.definition, })); const prepared = this.preparedIndex(fingerprint, catalog); - pending = (async (): Promise => { + const controller = new AbortController(); + const routingSignal = this.options.signal + ? AbortSignal.any([this.options.signal, controller.signal]) + : controller.signal; + const promise = (async (): Promise => { const result = await routeCompositionalTools(query, catalog, { ...this.options, preparedIndex: prepared.promise, - // Requests cancel only their own wait; shared work stays usable. - signal: undefined, + signal: routingSignal, }); const selected = result.disposition === "passthrough" @@ -315,17 +332,34 @@ export class CompositionalToolRoutingTransform { }, }; })(); - this.routes.set(routeKey, pending); + work = { promise, controller, waiterCount: 0, settled: false }; + this.routes.set(routeKey, work); const runKeys = this.routeKeysByRun.get(runId) ?? new Set(); runKeys.add(routeKey); this.routeKeysByRun.set(runId, runKeys); - void pending.catch(() => { - if (this.routes.get(routeKey) === pending) this.routes.delete(routeKey); - this.routeKeysByRun.get(runId)?.delete(routeKey); - }); + const createdWork = work; + void promise.then( + () => { + createdWork.settled = true; + }, + () => { + createdWork.settled = true; + this.evictRoute(runId, routeKey, createdWork); + }, + ); } - const cached = await awaitWithAbort(pending, input.signal); + work.waiterCount += 1; + let cached: CachedRoute; + try { + cached = await awaitWithAbort(work.promise, input.signal); + } finally { + work.waiterCount -= 1; + if (work.waiterCount === 0 && !work.settled) { + work.controller.abort(); + this.evictRoute(runId, routeKey, work); + } + } if (cacheHit) cached.evidence.cache_hits += 1; if (cached.evidence.routing.fallback !== null) { diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index b1995bf39c..76d638496f 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -38,6 +38,11 @@ const MODES = ["progressive", "direct"] as const satisfies readonly ToolDisclosu const MCP_SERVER_NAME = "performance-test"; const MCP_TOKEN_ENV = "TOOL_DISCLOSURE_PERFORMANCE_TEST_MCP_TOKEN"; const TEST_TIMEOUT_MS = 50 * 60_000; +const ROUTING_DECOMPOSER_MAX_ATTEMPTS = 2; +const ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS = 120_000; +const ROUTE_ONLY_RUN_TIMEOUT_MS = 900_000; +const ROUTED_PROXY_REQUEST_TIMEOUT_MS = 720_000; +const ROUTED_AGENT_INVOCATION_TIMEOUT_MS = 900_000; function sandboxName(mode: ToolDisclosureMode): string { return `e2e-tool-disclosure-performance-${mode}`; @@ -86,10 +91,11 @@ test.skipIf(!shouldRunLiveE2E())( allow_remote: true, reasoning_control: "enable_thinking_false", json_object_response: true, - max_attempts: 2, + max_attempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, }, embedding: { kind: "portable" }, - timeout_ms: 180_000, + timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, + run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, }); } finally { previousRoutingCredential === undefined @@ -274,8 +280,8 @@ test.skipIf(!shouldRunLiveE2E())( allowRemote: true, reasoningControl: "enable_thinking_false", jsonObjectResponse: true, - maxAttempts: 2, - timeoutMs: 180_000, + maxAttempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, + timeoutMs: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, }), embedder: new PortableHashingTextEmbedder(), isRoutableTool: (name) => routedToolNames.has(name), @@ -284,6 +290,7 @@ test.skipIf(!shouldRunLiveE2E())( const routedProxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl: hosted.endpointUrl, allowRemoteHttpsUpstream: true, + requestTimeoutMs: ROUTED_PROXY_REQUEST_TIMEOUT_MS, requestTransform: routedTransform.requestTransform, }); const routedProxyAddress = await routedProxy.start(); @@ -351,7 +358,7 @@ test.skipIf(!shouldRunLiveE2E())( artifactName: "invoke-tool-disclosure-compositional", env: hostEnv(), redactionValues: [...driver.redactions, hosted.apiKey, bearerToken], - timeoutMs: 10 * 60_000, + timeoutMs: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, }); } finally { calls = mcp.endRun(); @@ -383,8 +390,11 @@ test.skipIf(!shouldRunLiveE2E())( routing_configuration: { reasoning_control: "enable_thinking_false", output_mode: "json-object", - max_attempts: 2, - timeout_ms: 180_000, + max_attempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, + timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, + route_only_run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, + proxy_request_timeout_ms: ROUTED_PROXY_REQUEST_TIMEOUT_MS, + agent_invocation_timeout_ms: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, embedding: "portable-lexical-hashing", }, route_evidence: routeEvidence, diff --git a/test/performance/tool-disclosure-compositional-routing-adapters.test.ts b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts index df6d763f06..7b821250df 100644 --- a/test/performance/tool-disclosure-compositional-routing-adapters.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts @@ -6,6 +6,9 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createOpenAIChatTaskDecomposer, createOpenAITextEmbedder, + MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS, + MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS, + MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS, type ModelUsageEvent, PortableHashingTextEmbedder, } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-adapters"; @@ -135,6 +138,178 @@ describe("independent compositional tool model adapters", () => { ]); }); + it("retries a schema-invalid success response and aggregates usage from both attempts", async () => { + const responses = [ + { + content: '{"subtasks":"not-an-array"}', + usage: { prompt_tokens: 7, completion_tokens: 3, total_tokens: 10 }, + }, + { + content: '{"subtasks":["retry succeeded"]}', + usage: { prompt_tokens: 8, completion_tokens: 4, total_tokens: 12 }, + }, + ]; + const fetchMock = vi.fn(async () => { + const response = responses[fetchMock.mock.calls.length - 1]; + return new Response( + JSON.stringify({ + choices: [{ message: { content: response.content } }], + usage: response.usage, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 2, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).resolves.toEqual(["retry succeeded"]); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(usage).toMatchObject([ + { + attempt: 1, + outcome: "failed", + prompt_tokens: 7, + completion_tokens: 3, + total_tokens: 10, + }, + { + attempt: 2, + outcome: "completed", + prompt_tokens: 8, + completion_tokens: 4, + total_tokens: 12, + }, + ]); + expect( + usage.reduce( + (totals, event) => ({ + prompt_tokens: totals.prompt_tokens + (event.prompt_tokens ?? 0), + completion_tokens: totals.completion_tokens + (event.completion_tokens ?? 0), + total_tokens: totals.total_tokens + (event.total_tokens ?? 0), + }), + { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + ), + ).toEqual({ prompt_tokens: 15, completion_tokens: 7, total_tokens: 22 }); + }); + + it("stops after the exact configured retry count is exhausted", async () => { + const fetchMock = vi.fn(async () => new Response("temporary failure", { status: 503 })); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 3, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).rejects.toThrow("decomposition request failed"); + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(usage).toMatchObject([ + { attempt: 1, outcome: "failed" }, + { attempt: 2, outcome: "failed" }, + { attempt: 3, outcome: "failed" }, + ]); + }); + + it("does not retry a non-retryable authentication response", async () => { + const fetchMock = vi.fn(async () => new Response("unauthorized", { status: 401 })); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 3, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).rejects.toThrow("decomposition request failed"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(usage).toMatchObject([{ attempt: 1, outcome: "failed" }]); + }); + + it("does not retry after its parent signal aborts", async () => { + const controller = new AbortController(); + const fetchMock = vi.fn(async () => { + controller.abort(); + throw new DOMException("aborted", "AbortError"); + }); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 3, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ + pass: "initial", + query: "request", + tool_hints: [], + signal: controller.signal, + }), + ).rejects.toThrow("decomposition request failed"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(usage).toMatchObject([{ attempt: 1, outcome: "failed" }]); + }); + + it("enforces attempt, request-timeout, and total decomposition budget ceilings", () => { + const baseOptions = { + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + } as const; + + expect(() => + createOpenAIChatTaskDecomposer({ + ...baseOptions, + maxAttempts: MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS + 1, + }), + ).toThrow(`maxAttempts must not exceed ${MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS}`); + expect(() => + createOpenAIChatTaskDecomposer({ + ...baseOptions, + timeoutMs: MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS + 1, + }), + ).toThrow(`timeoutMs must not exceed ${MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS}`); + expect(() => + createOpenAITextEmbedder({ + ...baseOptions, + timeoutMs: MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS + 1, + }), + ).toThrow(`timeoutMs must not exceed ${MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS}`); + expect(() => + createOpenAIChatTaskDecomposer({ + ...baseOptions, + maxAttempts: MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS, + timeoutMs: + Math.floor( + MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS / MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS, + ) + 1, + }), + ).toThrow( + `timeoutMs * maxAttempts must not exceed ${MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS}`, + ); + }); + it("batches embeddings and restores provider results to input order", async () => { const observedInputs: string[][] = []; vi.stubGlobal( diff --git a/test/performance/tool-disclosure-compositional-routing-run.test.ts b/test/performance/tool-disclosure-compositional-routing-run.test.ts index 31170a4331..56281305db 100644 --- a/test/performance/tool-disclosure-compositional-routing-run.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-run.test.ts @@ -57,6 +57,7 @@ describe("compositional routing acceptance runner", () => { max_attempts: 2, }, embedding: { kind: "portable", dimensions: 1_024 }, + run_timeout_ms: 900_000, }); expect(output).toMatchObject({ @@ -69,6 +70,7 @@ describe("compositional routing acceptance runner", () => { decomposer_output_mode: "json-object", decomposer_max_attempts: 2, request_timeout_ms: 120_000, + run_timeout_ms: 900_000, embedding_kind: "portable", embedding_model: "portable-lexical-hashing", embedding_revision: "builtin-v1", @@ -87,10 +89,16 @@ describe("compositional routing acceptance runner", () => { }, embedding: { requests: 0, failed_requests: 0 }, }, + execution: { + status: "completed", + completed_case_count: 8, + total_case_count: 8, + }, acceptance_passed: true, }); expect(output.cases).toHaveLength(8); expect(output.cases.every((entry) => entry.shared_initial_decomposition)).toBe(true); + expect(output.cases.every((entry) => entry.evaluation_status === "completed")).toBe(true); const noTool = output.cases.find((entry) => entry.case_id === "route-no-tool-01"); expect(noTool?.refined).toMatchObject({ disposition: "routed", @@ -164,4 +172,100 @@ describe("compositional routing acceptance runner", () => { }); expect(noTool?.refined.forwarded_tool_names).toHaveLength(20); }); + + it("rejects request and overall timeouts outside their supported bounds", async () => { + const baseConfig = { + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + allow_remote: true, + }, + embedding: { kind: "portable" as const }, + }; + + await expect( + runCompositionalRoutingAcceptance({ ...baseConfig, timeout_ms: 300_001 }), + ).rejects.toThrow("timeout_ms must be a positive safe integer no greater than 300000"); + await expect( + runCompositionalRoutingAcceptance({ ...baseConfig, run_timeout_ms: 2_700_001 }), + ).rejects.toThrow("run_timeout_ms must be a positive safe integer no greater than 2700000"); + }); + + it("uses the overall deadline to stop outstanding decomposition work", async () => { + const fetchMock = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const signal = init?.signal as AbortSignal; + signal.throwIfAborted(); + return new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError")), { + once: true, + }); + }); + }); + vi.stubGlobal("fetch", fetchMock); + + const output = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + allow_remote: true, + max_attempts: 3, + }, + embedding: { kind: "portable" }, + run_timeout_ms: 10, + }); + + expect(output.acceptance_passed).toBe(false); + expect(output.configuration.run_timeout_ms).toBe(10); + expect(output.execution).toEqual({ + status: "timed-out", + completed_case_count: 0, + total_case_count: 8, + }); + expect(output.comparison.refined.route_failure_case_count).toBe(8); + expect(output.comparison.reasons).toContain("run deadline exceeded after 0 of 8 cases"); + expect( + output.cases.every( + (entry) => + entry.evaluation_status === "run-deadline-exceeded" && + entry.initial.evidence.fallback === "run-deadline-exceeded" && + entry.refined.evidence.fallback === "run-deadline-exceeded", + ), + ).toBe(true); + expect(output.usage.decomposition).toMatchObject({ requests: 1, failed_requests: 1 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("cannot pass after the monotonic run deadline when timer callbacks are delayed", async () => { + let monotonicNow = 0; + vi.spyOn(performance, "now").mockImplementation(() => { + monotonicNow += 2; + return monotonicNow; + }); + const fetchMock = vi.fn( + async () => + new Response(JSON.stringify({ choices: [{ message: { content: '{"subtasks":[]}' } }] }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + + const output = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + allow_remote: true, + }, + embedding: { kind: "portable" }, + run_timeout_ms: 1, + }); + + expect(output.acceptance_passed).toBe(false); + expect(output.execution.status).toBe("timed-out"); + expect(output.comparison.reasons).toContain("run deadline exceeded after 0 of 8 cases"); + expect(fetchMock).not.toHaveBeenCalled(); + }); }); diff --git a/test/performance/tool-disclosure-compositional-routing-transform.test.ts b/test/performance/tool-disclosure-compositional-routing-transform.test.ts index fd52af982f..93b9b7146b 100644 --- a/test/performance/tool-disclosure-compositional-routing-transform.test.ts +++ b/test/performance/tool-disclosure-compositional-routing-transform.test.ts @@ -355,13 +355,68 @@ describe("independent compositional tool request transform", () => { releaseInitial?.(["list calendar events"]); expect(names(await second)).toEqual(["core_shell", "route_calendar_list"]); - expect(decompositionSignals).toEqual([undefined, undefined]); - expect(embeddingSignals.every((signal) => signal === undefined)).toBe(true); + const [routeSignal] = decompositionSignals; + expect(routeSignal).toBeDefined(); + expect(decompositionSignals.every((signal) => signal === routeSignal)).toBe(true); + expect(routeSignal?.aborted).toBe(false); + expect(embeddingSignals[0]).toBeUndefined(); + expect(embeddingSignals.slice(1).every((signal) => signal === routeSignal)).toBe(true); expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ { cache_hits: 1, routing: { fallback: null } }, ]); }); + it("cancels unfinished route work after its last waiter aborts and starts fresh work", async () => { + let initialCalls = 0; + let firstRouteSignal: AbortSignal | undefined; + let markFirstStarted: (() => void) | undefined; + const firstStarted = new Promise((resolve) => { + markFirstStarted = resolve; + }); + const transform = new CompositionalToolRoutingTransform({ + decomposer: { + decompose: async (request) => { + initialCalls += request.pass === "initial" ? 1 : 0; + const holdFirst = request.pass === "initial" && initialCalls === 1; + return holdFirst + ? ((firstRouteSignal = request.signal), + markFirstStarted?.(), + new Promise((_resolve, reject) => { + request.signal?.addEventListener( + "abort", + () => reject(new DOMException("routing aborted", "AbortError")), + { once: true }, + ); + })) + : ["list calendar events"]; + }, + }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const body = requestBody("List today's calendar events."); + const controller = new AbortController(); + const first = transform.requestTransform({ + ...input(body), + signal: controller.signal, + }); + await firstStarted; + const firstRejection = expect(first).rejects.toMatchObject({ name: "AbortError" }); + + controller.abort(); + await firstRejection; + + expect(firstRouteSignal?.aborted).toBe(true); + expect(await transform.consumeEvidence("route-test-run")).toEqual([]); + + const fresh = await transform.requestTransform(input(body, 2)); + expect(names(fresh)).toEqual(["core_shell", "route_calendar_list"]); + expect(initialCalls).toBe(2); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { cache_hits: 0, routing: { fallback: null } }, + ]); + }); + it("preserves a request whose named tool choice conflicts with the route", async () => { const transform = new CompositionalToolRoutingTransform({ decomposer: { decompose: async () => ["list calendar events"] }, From 1bf260139b8181814a90f30613c2745c12b0bfa2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 09:49:06 -0700 Subject: [PATCH 23/31] fix(performance): route smoke through authenticated bridge Signed-off-by: Aaron Erickson --- ...ssive-tool-disclosure-performance-test.mdx | 2 + scripts/performance/README.md | 6 +- .../performance/tool-disclosure/recorder.ts | 99 +++++++-- .../tool-disclosure-performance-smoke.test.ts | 124 +++++++++++- .../tool-disclosure-recorder.test.ts | 190 +++++++++++++++++- 5 files changed, 388 insertions(+), 33 deletions(-) diff --git a/docs/inference/progressive-tool-disclosure-performance-test.mdx b/docs/inference/progressive-tool-disclosure-performance-test.mdx index 49eb65e393..e652bd08ac 100644 --- a/docs/inference/progressive-tool-disclosure-performance-test.mdx +++ b/docs/inference/progressive-tool-disclosure-performance-test.mdx @@ -19,6 +19,7 @@ Run both campaigns and retain their evidence before you make a performance or qu The explicit-only `tool-disclosure-performance-smoke` E2E job is a smaller operational check for ordinary GitHub-hosted runners. It runs the route-only acceptance corpus, one frozen task in progressive and direct modes with fresh Deep Agents sandboxes, and a separate routed replay of that task through the request transformer. +The routed replay keeps its recorder on a host-local private Docker bridge within the trusted runner boundary, rejects requests before transformation unless they carry a fresh ephemeral credential, and replaces that credential with the hosted provider credential only on the runner side. Its output validates live wiring only: it is not a complete campaign, cannot be summarized by this protocol, and does not support performance claims. ## Exercise the compositional routing experiment @@ -111,6 +112,7 @@ It does not implement or claim a dependency-graph planner or compatibility compo The route-only command does not execute an agent or a tool; use the full performance procedure below to validate task execution and end-to-end behavior. It also does not install the router into the frozen direct/progressive campaign. The opt-in smoke job exercises the transform in one separate, claim-ineligible agent replay and requires successful task execution, the expected tool call, no routing fallback, and fewer model-visible schemas. +That ordinary-runner bridge is test-only; the complete two-campaign procedure keeps the recorder on its documented local host boundary and does not route measurements through the smoke bridge. A broader agent-level routing comparison must use a separate initial/refined schedule in which both agent cells expose the same complete catalog before the request transformer runs. ## Understand the frozen protocol diff --git a/scripts/performance/README.md b/scripts/performance/README.md index a26843a2d4..3a96a398eb 100644 --- a/scripts/performance/README.md +++ b/scripts/performance/README.md @@ -79,7 +79,11 @@ evidence required for that result. The explicit live smoke also runs a separate routed replay after its frozen direct/progressive cells. That replay must preserve task correctness and the expected tool call while reducing model-visible schemas without fallback. It -does not alter either frozen cell and remains claim-ineligible. +does not alter either frozen cell and remains claim-ineligible. On an ordinary +GitHub runner, its recorder uses an authenticated, host-local private Docker +bridge inside the trusted runner boundary. Its ephemeral ingress credential is +replaced before upstream forwarding; full campaigns retain the protocol's +local recorder topology. Run the route-only corpus with a real decomposer using: diff --git a/scripts/performance/tool-disclosure/recorder.ts b/scripts/performance/tool-disclosure/recorder.ts index 17712663db..7a793d55a7 100644 --- a/scripts/performance/tool-disclosure/recorder.ts +++ b/scripts/performance/tool-disclosure/recorder.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; import http, { type IncomingHttpHeaders, type IncomingMessage, @@ -92,10 +92,15 @@ export interface ToolDisclosureRecordingEvent { export interface RecordingProxyOptions { upstreamBaseUrl: string; - /** Explicitly allow a credential-free non-loopback HTTPS upstream. */ + /** Explicitly allow a fixed non-loopback HTTPS upstream without URL-embedded credentials. */ allowRemoteHttpsUpstream?: boolean; - /** Must be exactly 127.0.0.1. Exposed so unsafe configuration fails closed. */ + /** Defaults to loopback. A private IPv4 bridge requires explicit opt-in and auth translation. */ listenHost?: string; + allowAuthenticatedPrivateIpv4Listener?: boolean; + /** Runtime-only exact bearer header checked before request reading, recording, or transformation. */ + requiredAuthorization?: string; + /** Runtime-only bearer header that replaces the accepted ingress credential upstream. */ + upstreamAuthorization?: string; /** Zero selects an ephemeral port. */ port?: number; maxRequestBodyBytes?: number; @@ -126,7 +131,7 @@ export type RecordingProxyRequestTransform = ( ) => Buffer | Promise; export interface RecordingProxyAddress { - host: typeof LOOPBACK_HOST; + host: string; port: number; base_url: string; } @@ -197,6 +202,32 @@ function validatePositiveInteger(value: number, label: string): void { } } +function isPrivateIpv4(value: string): boolean { + if (isIP(value) !== 4) return false; + const [first, second] = value.split(".").map(Number); + return ( + first === 10 || + (first === 172 && second >= 16 && second <= 31) || + (first === 192 && second === 168) + ); +} + +function authorizationHeader(value: string | undefined, label: string): string | undefined { + if (value === undefined) return undefined; + const token = value.startsWith("Bearer ") ? value.slice("Bearer ".length) : ""; + if (!token || token.length > 4_096 || /\s/u.test(token)) { + throw new Error(`${label} must be a bounded Bearer authorization header`); + } + return value; +} + +function authorizationMatches(value: string | undefined, expected: string): boolean { + if (value === undefined) return false; + const actualDigest = createHash("sha256").update(value).digest(); + const expectedDigest = createHash("sha256").update(expected).digest(); + return timingSafeEqual(actualDigest, expectedDigest); +} + function parseUpstreamBaseUrl(value: string, allowRemoteHttpsUpstream: boolean): URL { let url: URL; try { @@ -323,6 +354,7 @@ function connectionHeaderNames(headers: IncomingHttpHeaders): Set { function forwardedRequestHeaders( headers: IncomingHttpHeaders, bodyLength: number, + authorization: string | undefined, ): OutgoingHttpHeaders { const result: OutgoingHttpHeaders = {}; const connectionNames = connectionHeaderNames(headers); @@ -340,6 +372,7 @@ function forwardedRequestHeaders( result[name] = value; } result["content-length"] = String(bodyLength); + if (authorization !== undefined) result.authorization = authorization; return result; } @@ -404,17 +437,21 @@ function readBoundedBody(request: IncomingMessage, maxBytes: number): Promise number; private readonly sockets = new Set(); private readonly events: ToolDisclosureRecordingEvent[] = []; @@ -424,8 +461,28 @@ export class ToolDisclosureRecordingProxy { constructor(options: RecordingProxyOptions) { const listenHost = options.listenHost ?? LOOPBACK_HOST; - if (listenHost !== LOOPBACK_HOST) { - throw new Error(`recording proxy listenHost must be exactly ${LOOPBACK_HOST}`); + const requiredAuthorization = authorizationHeader( + options.requiredAuthorization, + "requiredAuthorization", + ); + const upstreamAuthorization = authorizationHeader( + options.upstreamAuthorization, + "upstreamAuthorization", + ); + if ((requiredAuthorization === undefined) !== (upstreamAuthorization === undefined)) { + throw new Error("recording proxy authorization translation requires both headers"); + } + if ( + listenHost !== LOOPBACK_HOST && + !( + options.allowAuthenticatedPrivateIpv4Listener === true && + isPrivateIpv4(listenHost) && + requiredAuthorization !== undefined + ) + ) { + throw new Error( + `recording proxy listenHost must be exactly ${LOOPBACK_HOST} unless an authenticated private IPv4 listener is explicitly enabled`, + ); } const port = options.port ?? 0; if (!Number.isSafeInteger(port) || port < 0 || port > 65_535) { @@ -446,6 +503,8 @@ export class ToolDisclosureRecordingProxy { this.requestTimeoutMs = requestTimeoutMs; this.requiredTemperature = options.requiredTemperature; this.requestTransform = options.requestTransform; + this.requiredAuthorization = requiredAuthorization; + this.upstreamAuthorization = upstreamAuthorization; this.clock = () => performance.now(); } @@ -483,14 +542,14 @@ export class ToolDisclosureRecordingProxy { this.server = server; const address = server.address(); - if (!address || typeof address === "string" || address.address !== LOOPBACK_HOST) { + if (!address || typeof address === "string" || address.address !== this.listenHost) { await this.stop(); - throw new Error("recording proxy failed to bind the required loopback address"); + throw new Error("recording proxy failed to bind the required listener address"); } return { - host: LOOPBACK_HOST, + host: this.listenHost, port: address.port, - base_url: `http://${LOOPBACK_HOST}:${address.port}`, + base_url: `http://${this.listenHost}:${address.port}`, }; } @@ -679,6 +738,14 @@ export class ToolDisclosureRecordingProxy { fixedResponse(response, 404, "not found"); return; } + if ( + this.requiredAuthorization !== undefined && + !authorizationMatches(request.headers.authorization, this.requiredAuthorization) + ) { + request.resume(); + fixedResponse(response, 401, "unauthorized"); + return; + } const endpoint = classifyEndpoint(incoming.pathname); const method = classifyMethod(request.method); @@ -853,7 +920,11 @@ export class ToolDisclosureRecordingProxy { target, { method: incomingRequest.method, - headers: forwardedRequestHeaders(incomingRequest.headers, body.length), + headers: forwardedRequestHeaders( + incomingRequest.headers, + body.length, + this.upstreamAuthorization, + ), }, (upstreamResponse) => { statusCode = upstreamResponse.statusCode ?? 502; diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 76d638496f..dc5a7f89aa 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -3,7 +3,6 @@ import { execFileSync } from "node:child_process"; import { randomBytes } from "node:crypto"; - import { DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, generateCatalogPrefix, @@ -25,6 +24,7 @@ import { startQuickTunnel } from "../../../scripts/performance/tool-disclosure/q import { createToolDisclosureRecordingProxy } from "../../../scripts/performance/tool-disclosure/recorder"; import type { ToolDisclosureMode } from "../../../scripts/performance/tool-disclosure/schedule"; import { generatePrimaryTaskSet } from "../../../scripts/performance/tool-disclosure/tasks"; +import { LOCAL_INFERENCE_TIMEOUT_SECS } from "../../../src/lib/onboard/env"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -37,12 +37,18 @@ const TASK_ID = "primary-single-01"; const MODES = ["progressive", "direct"] as const satisfies readonly ToolDisclosureMode[]; const MCP_SERVER_NAME = "performance-test"; const MCP_TOKEN_ENV = "TOOL_DISCLOSURE_PERFORMANCE_TEST_MCP_TOKEN"; +const OPENSHELL_DOCKER_NETWORK = + process.env.OPENSHELL_DOCKER_NETWORK_NAME?.trim() || "openshell-docker"; +const PRIVATE_BRIDGE_PROBE_IMAGE = + "busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662"; const TEST_TIMEOUT_MS = 50 * 60_000; const ROUTING_DECOMPOSER_MAX_ATTEMPTS = 2; const ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS = 120_000; const ROUTE_ONLY_RUN_TIMEOUT_MS = 900_000; const ROUTED_PROXY_REQUEST_TIMEOUT_MS = 720_000; +const ROUTED_GATEWAY_REQUEST_TIMEOUT_MS = 840_000; const ROUTED_AGENT_INVOCATION_TIMEOUT_MS = 900_000; +const RESTORED_GATEWAY_REQUEST_TIMEOUT_MS = LOCAL_INFERENCE_TIMEOUT_SECS * 1_000; function sandboxName(mode: ToolDisclosureMode): string { return `e2e-tool-disclosure-performance-${mode}`; @@ -75,7 +81,13 @@ test.skipIf(!shouldRunLiveE2E())( async ({ artifacts, cleanup, host, sandbox, secrets }) => { const hosted = requireHostedInferenceConfig(secrets); const bearerToken = randomBytes(32).toString("hex"); - artifacts.addRedactionValues([hosted.apiKey, bearerToken]); + const routingIngressToken = randomBytes(32).toString("hex"); + artifacts.addRedactionValues([ + hosted.apiKey, + hosted.endpointUrl, + bearerToken, + routingIngressToken, + ]); const routingCredentialEnv = "NEMOCLAW_COMPOSITIONAL_ROUTING_SMOKE_API_KEY"; const previousRoutingCredential = process.env[routingCredentialEnv]; @@ -287,16 +299,75 @@ test.skipIf(!shouldRunLiveE2E())( isRoutableTool: (name) => routedToolNames.has(name), requireRouting: true, }); + const networkInspect = await host.command( + "docker", + ["network", "inspect", "--format", "{{json .IPAM.Config}}", OPENSHELL_DOCKER_NETWORK], + { + artifactName: "compositional-routing-private-network", + env: hostEnv(), + timeoutMs: 30_000, + }, + ); + expect(networkInspect.exitCode, resultText(networkInspect)).toBe(0); + const networkIpam = JSON.parse(networkInspect.stdout) as Array<{ + Gateway?: unknown; + }>; + const privateListenHost = networkIpam + .map((entry) => entry.Gateway) + .find((gateway): gateway is string => typeof gateway === "string" && gateway.includes(".")); + expect(privateListenHost, "OpenShell Docker network has no IPv4 gateway").toBeDefined(); + const recorderListenHost = privateListenHost as string; const routedProxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl: hosted.endpointUrl, allowRemoteHttpsUpstream: true, + listenHost: recorderListenHost, + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: `Bearer ${routingIngressToken}`, + upstreamAuthorization: `Bearer ${hosted.apiKey}`, requestTimeoutMs: ROUTED_PROXY_REQUEST_TIMEOUT_MS, requestTransform: routedTransform.requestTransform, }); const routedProxyAddress = await routedProxy.start(); cleanup.add("stop compositional routing recording proxy", () => routedProxy.stop()); + expect(routedProxyAddress.host).toBe(recorderListenHost); + expect(routedProxyAddress.port).toBeGreaterThanOrEqual(1_024); + + const bridgeProbe = await host.command( + "docker", + [ + "run", + "--rm", + "--pull=missing", + "--network", + OPENSHELL_DOCKER_NETWORK, + PRIVATE_BRIDGE_PROBE_IMAGE, + "nc", + "-zw5", + recorderListenHost, + String(routedProxyAddress.port), + ], + { + artifactName: "compositional-routing-private-reachability", + env: hostEnv(), + redactionValues: [routedProxyAddress.base_url], + timeoutMs: 30_000, + }, + ); + expect( + bridgeProbe.exitCode, + `OpenShell Docker network cannot reach the authenticated routing bridge: ${resultText(bridgeProbe)}`, + ).toBe(0); - const configureInferenceRoute = async (baseUrl: string, artifactSuffix: string) => { + const configureInferenceRoute = async ( + baseUrl: string, + artifactSuffix: string, + credential: string, + timeoutMs: number, + ) => { + artifacts.addRedactionValues([baseUrl, credential]); + const providerEnv = hostEnv({ + [hosted.credentialEnv]: credential, + }); const update = await host.command( "openshell", [ @@ -310,8 +381,8 @@ test.skipIf(!shouldRunLiveE2E())( ], { artifactName: `compositional-routing-provider-${artifactSuffix}`, - env: hostEnv(hosted.env), - redactionValues: [hosted.apiKey], + env: providerEnv, + redactionValues: [hosted.apiKey, routingIngressToken, baseUrl, credential], timeoutMs: 2 * 60_000, }, ); @@ -326,20 +397,41 @@ test.skipIf(!shouldRunLiveE2E())( hosted.providerName, "--model", hosted.model, + "--timeout", + String(timeoutMs / 1_000), ], { artifactName: `compositional-routing-select-${artifactSuffix}`, - env: hostEnv(hosted.env), - redactionValues: [hosted.apiKey], + env: providerEnv, + redactionValues: [hosted.apiKey, routingIngressToken, baseUrl, credential], timeoutMs: 2 * 60_000, }, ); expect(select.exitCode, resultText(select)).toBe(0); }; + let providerRestore: Promise | undefined; + const restoreInferenceRoute = (): Promise => + providerRestore ?? + (providerRestore = configureInferenceRoute( + hosted.endpointUrl, + "restore", + hosted.apiKey, + RESTORED_GATEWAY_REQUEST_TIMEOUT_MS, + ).catch((error: unknown) => { + providerRestore = undefined; + throw error; + })); + cleanup.add("restore hosted inference route", restoreInferenceRoute); + let routedReplay: Record | undefined; try { - await configureInferenceRoute(`${routedProxyAddress.base_url}/v1`, "enable"); + await configureInferenceRoute( + `${routedProxyAddress.base_url}/v1`, + "enable", + routingIngressToken, + ROUTED_GATEWAY_REQUEST_TIMEOUT_MS, + ); const driver = buildAgentDriverCommand({ openshellBin: process.env.OPENSHELL_BIN, sandboxName: sandboxName("direct"), @@ -357,7 +449,13 @@ test.skipIf(!shouldRunLiveE2E())( invocation = await host.command(driver.command, driver.args, { artifactName: "invoke-tool-disclosure-compositional", env: hostEnv(), - redactionValues: [...driver.redactions, hosted.apiKey, bearerToken], + redactionValues: [ + ...driver.redactions, + hosted.apiKey, + bearerToken, + routingIngressToken, + routedProxyAddress.base_url, + ], timeoutMs: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, }); } finally { @@ -394,7 +492,10 @@ test.skipIf(!shouldRunLiveE2E())( timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, route_only_run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, proxy_request_timeout_ms: ROUTED_PROXY_REQUEST_TIMEOUT_MS, + gateway_request_timeout_ms: ROUTED_GATEWAY_REQUEST_TIMEOUT_MS, agent_invocation_timeout_ms: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, + baseline_gateway_request_timeout_ms: RESTORED_GATEWAY_REQUEST_TIMEOUT_MS, + gateway_transport: "authenticated-private-host-bridge", embedding: "portable-lexical-hashing", }, route_evidence: routeEvidence, @@ -402,11 +503,14 @@ test.skipIf(!shouldRunLiveE2E())( limitations: [ "This single routed replay is an end-to-end wiring check, not a performance or quality claim.", "The portable lexical embedder is used only for an ordinary-runner smoke path.", + "The replay uses an authenticated private runner bridge and replaces its ephemeral ingress credential before the hosted request.", + "The bridge uses host-local HTTP inside the trusted runner and Docker-network boundary.", + "Proxy and gateway timeouts apply per request; the routed agent invocation has its own overall bound.", "The routed replay is separate from the frozen direct/progressive comparison.", ], }; } finally { - await configureInferenceRoute(hosted.endpointUrl, "restore"); + await restoreInferenceRoute(); } await artifacts.writeJson("compositional-routing-agent-smoke.json", routedReplay); diff --git a/test/performance/tool-disclosure-recorder.test.ts b/test/performance/tool-disclosure-recorder.test.ts index c77973def1..a057c366d3 100644 --- a/test/performance/tool-disclosure-recorder.test.ts +++ b/test/performance/tool-disclosure-recorder.test.ts @@ -54,13 +54,20 @@ async function startUpstream( async function startProxy( upstreamBaseUrl: string, options: { + listenHost?: string; + allowAuthenticatedPrivateIpv4Listener?: boolean; + requiredAuthorization?: string; + upstreamAuthorization?: string; maxRequestBodyBytes?: number; requestTimeoutMs?: number; requiredTemperature?: number; requestTransform?: RecordingProxyRequestTransform; } = {}, ): Promise<{ proxy: ToolDisclosureRecordingProxy; baseUrl: string }> { - const proxy = createToolDisclosureRecordingProxy({ upstreamBaseUrl, ...options }); + const proxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl, + ...options, + }); const address = await proxy.start(); cleanup.push(() => proxy.stop()); expect(address.host).toBe("127.0.0.1"); @@ -100,6 +107,87 @@ describe("tool-disclosure recording proxy", () => { upstreamBaseUrl: "http://inference.example/v1", }), ).toThrow("upstreamBaseUrl is allowed only on loopback"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + listenHost: "172.18.0.1", + allowAuthenticatedPrivateIpv4Listener: true, + }), + ).toThrow("authenticated private IPv4 listener"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + listenHost: "203.0.113.10", + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: "Bearer ingress-token", + upstreamAuthorization: "Bearer upstream-token", + }), + ).toThrow("must be exactly 127.0.0.1"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + listenHost: "172.18.0.1", + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: "Bearer ingress-token", + upstreamAuthorization: "Bearer upstream-token", + }), + ).not.toThrow(); + }); + + it("requires paired, bounded bearer headers for authenticated private listeners", () => { + const privateListener = { + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + listenHost: "172.18.0.1", + allowAuthenticatedPrivateIpv4Listener: true, + } as const; + expect(() => + createToolDisclosureRecordingProxy({ + ...privateListener, + requiredAuthorization: "Bearer ingress-token", + }), + ).toThrow("requires both headers"); + expect(() => + createToolDisclosureRecordingProxy({ + ...privateListener, + upstreamAuthorization: "Bearer upstream-token", + }), + ).toThrow("requires both headers"); + for (const malformed of [ + "Basic token", + "Bearer ", + "Bearer token with spaces", + `Bearer ${"x".repeat(4_097)}`, + ]) { + expect(() => + createToolDisclosureRecordingProxy({ + ...privateListener, + requiredAuthorization: malformed, + upstreamAuthorization: "Bearer upstream-token", + }), + ).toThrow("bounded Bearer authorization header"); + } + }); + + it("accepts only RFC1918 address ranges for authenticated private listeners", () => { + const options = { + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: "Bearer ingress-token", + upstreamAuthorization: "Bearer upstream-token", + } as const; + for (const listenHost of ["10.0.0.1", "172.16.0.1", "172.31.255.254", "192.168.1.1"]) { + expect(() => createToolDisclosureRecordingProxy({ ...options, listenHost })).not.toThrow(); + } + for (const listenHost of ["172.15.255.254", "172.32.0.1", "192.167.1.1", "192.169.1.1"]) { + expect(() => createToolDisclosureRecordingProxy({ ...options, listenHost })).toThrow( + "must be exactly 127.0.0.1", + ); + } }); it("rejects non-loopback HTTPS recorder upstreams for the local performance test", () => { @@ -186,7 +274,11 @@ describe("tool-disclosure recording proxy", () => { }); it("forwards requests while retaining only canonical tool metadata", async () => { - const received: Array<{ url: string; authorization: string; body: string }> = []; + const received: Array<{ + url: string; + authorization: string; + body: string; + }> = []; const upstream = await startUpstream((request, response) => { void collectBody(request).then((body) => { received.push({ @@ -313,6 +405,72 @@ describe("tool-disclosure recording proxy", () => { expect(proxy.getEvents()).toEqual([]); }); + it("authenticates before recording or transformation and replaces the upstream credential", async () => { + const upstreamAuthorizations: string[] = []; + const upstream = await startUpstream((request, response) => { + upstreamAuthorizations.push(String(request.headers.authorization ?? "")); + response.end("{}"); + }); + let transformCalls = 0; + const ingressAuthorization = "Bearer private-ingress-token"; + const upstreamAuthorization = "Bearer private-upstream-token"; + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requiredAuthorization: ingressAuthorization, + upstreamAuthorization, + requestTransform: ({ body }) => { + transformCalls += 1; + return body; + }, + }); + + proxy.beginRun("authorization-translation"); + const missing = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[],"tools":[]}', + }); + expect(missing.status).toBe(401); + expect(await missing.json()).toEqual({ error: "unauthorized" }); + const rejected = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { + authorization: "Bearer private-ingress-tokeN", + "content-type": "application/json", + }, + body: '{"messages":[],"tools":[]}', + }); + expect(rejected.status).toBe(401); + expect(await rejected.json()).toEqual({ error: "unauthorized" }); + expect(transformCalls).toBe(0); + expect(upstreamAuthorizations).toEqual([]); + + const acceptedBody = '{"messages":[],"tools":[]}'; + const acceptedStatus = await new Promise((resolve, reject) => { + const request = http.request(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { + authorization: ingressAuthorization, + connection: "authorization", + "content-length": Buffer.byteLength(acceptedBody), + "content-type": "application/json", + }, + }); + request.once("response", (response) => { + response.resume(); + response.once("end", () => resolve(response.statusCode ?? 0)); + }); + request.once("error", reject); + request.end(acceptedBody); + }); + expect(acceptedStatus).toBe(200); + + const events = proxy.endRun(); + expect(transformCalls).toBe(1); + expect(upstreamAuthorizations).toEqual([upstreamAuthorization]); + expect(events).toHaveLength(1); + expect(JSON.stringify(events)).not.toMatch(/private-ingress-token|private-upstream-token/); + }); + it("applies an async request transform before forwarding and records transformed tools", async () => { let receivedBody = ""; const upstream = await startUpstream((request, response) => { @@ -342,18 +500,28 @@ describe("tool-disclosure recording proxy", () => { await Promise.resolve(); return Buffer.from(JSON.stringify(payload), "utf8"); }; - const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { requestTransform }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTransform, + }); const requestBody = JSON.stringify({ model: "private-model", messages: [{ role: "user", content: "private-prompt" }], tools: [ { type: "function", - function: { name: "first_tool", description: "first-private", parameters: {} }, + function: { + name: "first_tool", + description: "first-private", + parameters: {}, + }, }, { type: "function", - function: { name: "second_tool", description: "second-private", parameters: {} }, + function: { + name: "second_tool", + description: "second-private", + parameters: {}, + }, }, ], }); @@ -518,7 +686,9 @@ describe("tool-disclosure recording proxy", () => { body: '{"messages":[],"tools":[]}', }); expect(response.status).toBe(400); - expect(await response.json()).toEqual({ error: "performance test temperature mismatch" }); + expect(await response.json()).toEqual({ + error: "performance test temperature mismatch", + }); expect(hits).toBe(0); expect(proxy.endRun()[0]).toMatchObject({ status_code: 400, @@ -584,7 +754,9 @@ describe("tool-disclosure recording proxy", () => { response.end("{}"); }); }); - const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { requiredTemperature: 0 }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requiredTemperature: 0, + }); proxy.beginRun("temperature-missing"); await fetch(`${baseUrl}/v1/chat/completions`, { method: "POST", @@ -625,7 +797,9 @@ describe("tool-disclosure recording proxy", () => { }); expect(response.status).toBe(502); expect(response.headers.get("location")).toBeNull(); - expect(await response.json()).toEqual({ error: "upstream redirect rejected" }); + expect(await response.json()).toEqual({ + error: "upstream redirect rejected", + }); expect(hits).toBe(1); const [event] = proxy.endRun(); From a8a2f41c7889a8e8f4961ecb5acc13223fc7a0ea Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 10:04:41 -0700 Subject: [PATCH 24/31] fix(performance): rely on project live gate Signed-off-by: Aaron Erickson --- .../tool-disclosure-performance-smoke.test.ts | 925 +++++++++--------- 1 file changed, 459 insertions(+), 466 deletions(-) diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index bfaf50c1c1..814895f066 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -74,504 +74,497 @@ function gitSha(): string { })(); } -test( - "tool disclosure hosted-inference performance smoke completes one frozen task in both modes", - { timeout: TEST_TIMEOUT_MS }, - async ({ artifacts, cleanup, host, sandbox, secrets }) => { - const hosted = requireHostedInferenceConfig(secrets); - const bearerToken = randomBytes(32).toString("hex"); - const routingIngressToken = randomBytes(32).toString("hex"); - artifacts.addRedactionValues([ - hosted.apiKey, - hosted.endpointUrl, - bearerToken, - routingIngressToken, - ]); +test("tool disclosure hosted-inference performance smoke completes one frozen task in both modes", { + timeout: TEST_TIMEOUT_MS, +}, async ({ artifacts, cleanup, host, sandbox, secrets }) => { + const hosted = requireHostedInferenceConfig(secrets); + const bearerToken = randomBytes(32).toString("hex"); + const routingIngressToken = randomBytes(32).toString("hex"); + artifacts.addRedactionValues([ + hosted.apiKey, + hosted.endpointUrl, + bearerToken, + routingIngressToken, + ]); - const routingCredentialEnv = "NEMOCLAW_COMPOSITIONAL_ROUTING_SMOKE_API_KEY"; - const previousRoutingCredential = process.env[routingCredentialEnv]; - process.env[routingCredentialEnv] = hosted.apiKey; - let routingAcceptance: Awaited>; - try { - routingAcceptance = await runCompositionalRoutingAcceptance({ - decomposer: { - base_url: hosted.endpointUrl, - model: hosted.model, - revision: process.env.NEMOCLAW_MODEL_REVISION?.trim() || "unreported", - api_key_env: routingCredentialEnv, - allow_remote: true, - reasoning_control: "enable_thinking_false", - json_object_response: true, - max_attempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, - }, - embedding: { kind: "portable" }, - timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, - run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, - }); - } finally { - previousRoutingCredential === undefined - ? delete process.env[routingCredentialEnv] - : (process.env[routingCredentialEnv] = previousRoutingCredential); - } - await artifacts.writeJson("compositional-routing-acceptance.json", routingAcceptance); - expect(routingAcceptance.acceptance_passed).toBe(true); - - const catalog = generateSyntheticCatalog({ - seed: DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, + const routingCredentialEnv = "NEMOCLAW_COMPOSITIONAL_ROUTING_SMOKE_API_KEY"; + const previousRoutingCredential = process.env[routingCredentialEnv]; + process.env[routingCredentialEnv] = hosted.apiKey; + let routingAcceptance: Awaited>; + try { + routingAcceptance = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: hosted.endpointUrl, + model: hosted.model, + revision: process.env.NEMOCLAW_MODEL_REVISION?.trim() || "unreported", + api_key_env: routingCredentialEnv, + allow_remote: true, + reasoning_control: "enable_thinking_false", + json_object_response: true, + max_attempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, + }, + embedding: { kind: "portable" }, + timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, + run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, }); - const catalogPrefix = generateCatalogPrefix(catalog, CATALOG_SIZE); - const primaryTasks = generatePrimaryTaskSet(catalog); - const task = primaryTasks.tasks.find((candidate) => candidate.id === TASK_ID); - expect(task, `frozen task ${TASK_ID} is missing`).toBeDefined(); - const frozenTask = task as NonNullable; - expect( - frozenTask.min_catalog_size, - `frozen task ${TASK_ID} does not fit the performance smoke catalog`, - ).toBeLessThanOrEqual(catalogPrefix.size); + } finally { + previousRoutingCredential === undefined + ? delete process.env[routingCredentialEnv] + : (process.env[routingCredentialEnv] = previousRoutingCredential); + } + await artifacts.writeJson("compositional-routing-acceptance.json", routingAcceptance); + expect(routingAcceptance.acceptance_passed).toBe(true); - await artifacts.writeJson("scenario.json", { - id: "tool-disclosure-performance-smoke", - boundary: - "tool-disclosure performance smoke with two real Deep Agents Code sandboxes, hosted inference, and synthetic MCP", - claim_eligible: false, - agent: AGENT, - modes: MODES, - catalog_size: CATALOG_SIZE, - task_id: TASK_ID, - model_id: hosted.model, - }); + const catalog = generateSyntheticCatalog({ + seed: DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, + }); + const catalogPrefix = generateCatalogPrefix(catalog, CATALOG_SIZE); + const primaryTasks = generatePrimaryTaskSet(catalog); + const task = primaryTasks.tasks.find((candidate) => candidate.id === TASK_ID); + expect(task, `frozen task ${TASK_ID} is missing`).toBeDefined(); + const frozenTask = task as NonNullable; + expect( + frozenTask.min_catalog_size, + `frozen task ${TASK_ID} does not fit the performance smoke catalog`, + ).toBeLessThanOrEqual(catalogPrefix.size); - const mcp = new SyntheticMcpServer(catalogPrefix, bearerToken); - const mcpAddress = await mcp.start(); - cleanup.add("stop tool-disclosure performance synthetic MCP server", () => mcp.stop()); - const tunnel = await startQuickTunnel({ port: mcpAddress.port }); - cleanup.add("stop tool-disclosure performance MCP quick tunnel", () => tunnel.close()); + await artifacts.writeJson("scenario.json", { + id: "tool-disclosure-performance-smoke", + boundary: + "tool-disclosure performance smoke with two real Deep Agents Code sandboxes, hosted inference, and synthetic MCP", + claim_eligible: false, + agent: AGENT, + modes: MODES, + catalog_size: CATALOG_SIZE, + task_id: TASK_ID, + model_id: hosted.model, + }); - for (const mode of MODES) { - const name = sandboxName(mode); - cleanup.add(`destroy ${mode} tool-disclosure performance smoke sandbox`, () => - host.bestEffortCleanupSandbox(name, { - artifactName: `cleanup-${mode}-tool-disclosure-sandbox`, - env: hostEnv(), - }), - ); - await host.cleanupSandbox(name, { - artifactName: `preclean-${mode}-tool-disclosure-sandbox`, - env: hostEnv(), - }); + const mcp = new SyntheticMcpServer(catalogPrefix, bearerToken); + const mcpAddress = await mcp.start(); + cleanup.add("stop tool-disclosure performance synthetic MCP server", () => mcp.stop()); + const tunnel = await startQuickTunnel({ port: mcpAddress.port }); + cleanup.add("stop tool-disclosure performance MCP quick tunnel", () => tunnel.close()); - const onboard = await host.nemoclaw( - [ - "onboard", - "--non-interactive", - "--fresh", - "--yes", - "--yes-i-accept-third-party-software", - "--agent", - AGENT, - "--name", - name, - "--tool-disclosure", - mode, - ], - { - artifactName: `onboard-tool-disclosure-${mode}`, - env: hostEnv({ - ...hosted.env, - NEMOCLAW_AGENT: AGENT, - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_SANDBOX_NAME: name, - NEMOCLAW_TOOL_DISCLOSURE: mode, - }), - redactionValues: [hosted.apiKey], - timeoutMs: 20 * 60_000, - }, - ); - expect(onboard.exitCode, resultText(onboard)).toBe(0); + for (const mode of MODES) { + const name = sandboxName(mode); + cleanup.add(`destroy ${mode} tool-disclosure performance smoke sandbox`, () => + host.bestEffortCleanupSandbox(name, { + artifactName: `cleanup-${mode}-tool-disclosure-sandbox`, + env: hostEnv(), + }), + ); + await host.cleanupSandbox(name, { + artifactName: `preclean-${mode}-tool-disclosure-sandbox`, + env: hostEnv(), + }); - const attestation = await sandbox.exec( + const onboard = await host.nemoclaw( + [ + "onboard", + "--non-interactive", + "--fresh", + "--yes", + "--yes-i-accept-third-party-software", + "--agent", + AGENT, + "--name", name, - [ - "nemoclaw-start", - "sh", - "-c", - `test "$NEMOCLAW_TOOL_DISCLOSURE" = ${JSON.stringify(mode)}`, - ], - { - artifactName: `attest-tool-disclosure-${mode}`, - env: sandboxAccessEnv(), - timeoutMs: 30_000, - }, - ); - expect(attestation.exitCode, resultText(attestation)).toBe(0); - } - - for (const mode of MODES) { - const add = await host.nemoclaw( - [ - sandboxName(mode), - "mcp", - "add", - MCP_SERVER_NAME, - "--url", - tunnel.mcpUrl, - "--env", - MCP_TOKEN_ENV, - ], - { - artifactName: `add-tool-disclosure-mcp-${mode}`, - env: hostEnv({ [MCP_TOKEN_ENV]: bearerToken }), - redactionValues: [bearerToken], - timeoutMs: 5 * 60_000, - }, - ); - expect(add.exitCode, resultText(add)).toBe(0); - } - - const results = []; - for (const mode of MODES) { - const runId = `performance-smoke-${AGENT}-${mode}-${TASK_ID}`; - const driver = buildAgentDriverCommand({ - openshellBin: process.env.OPENSHELL_BIN, - sandboxName: sandboxName(mode), - agent: AGENT, - prompt: frozenTask.prompt, - sessionId: runId, - }); - mcp.beginRun(runId); - const startedAt = process.hrtime.bigint(); - let invocation; - let calls; - try { - invocation = await host.command(driver.command, driver.args, { - artifactName: `invoke-tool-disclosure-${mode}`, - env: hostEnv(), - redactionValues: [...driver.redactions, hosted.apiKey, bearerToken], - timeoutMs: 10 * 60_000, - }); - } finally { - calls = mcp.endRun(); - } - const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); - const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); - const graded = gradeTaskRun(frozenTask, calls, finalOutput); - const outcome = invocation.timedOut - ? "timeout" - : invocation.exitCode === 0 - ? graded.outcome - : "model-error"; - results.push({ - run_id: runId, + "--tool-disclosure", mode, - outcome, - invocation: { - exit_code: invocation.exitCode, - timed_out: invocation.timedOut, - elapsed_ms: elapsedMs, - }, - synthetic_call_count: calls.length, - correctness: graded.correctness, - }); - } - - const routedRunId = `performance-smoke-${AGENT}-compositional-${TASK_ID}`; - const routedToolNames = new Set( - catalogPrefix.tools.map((tool) => `${MCP_SERVER_NAME}_${tool.definition.function.name}`), + ], + { + artifactName: `onboard-tool-disclosure-${mode}`, + env: hostEnv({ + ...hosted.env, + NEMOCLAW_AGENT: AGENT, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: name, + NEMOCLAW_TOOL_DISCLOSURE: mode, + }), + redactionValues: [hosted.apiKey], + timeoutMs: 20 * 60_000, + }, ); - const expectedRoutedToolName = `${MCP_SERVER_NAME}_${frozenTask.expected_calls[0]?.tool_name ?? ""}`; - expect( - routedToolNames.has(expectedRoutedToolName), - "the routed replay target is not present in the reviewed catalog", - ).toBe(true); - const routedTransform = new CompositionalToolRoutingTransform({ - decomposer: createOpenAIChatTaskDecomposer({ - baseUrl: hosted.endpointUrl, - model: hosted.model, - apiKey: hosted.apiKey, - allowRemote: true, - reasoningControl: "enable_thinking_false", - jsonObjectResponse: true, - maxAttempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, - timeoutMs: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, - }), - embedder: new PortableHashingTextEmbedder(), - isRoutableTool: (name) => routedToolNames.has(name), - requireRouting: true, - }); - const networkInspect = await host.command( - "docker", - ["network", "inspect", "--format", "{{json .IPAM.Config}}", OPENSHELL_DOCKER_NETWORK], + expect(onboard.exitCode, resultText(onboard)).toBe(0); + + const attestation = await sandbox.exec( + name, + ["nemoclaw-start", "sh", "-c", `test "$NEMOCLAW_TOOL_DISCLOSURE" = ${JSON.stringify(mode)}`], { - artifactName: "compositional-routing-private-network", - env: hostEnv(), + artifactName: `attest-tool-disclosure-${mode}`, + env: sandboxAccessEnv(), timeoutMs: 30_000, }, ); - expect(networkInspect.exitCode, resultText(networkInspect)).toBe(0); - const networkIpam = JSON.parse(networkInspect.stdout) as Array<{ - Gateway?: unknown; - }>; - const privateListenHost = networkIpam - .map((entry) => entry.Gateway) - .find((gateway): gateway is string => typeof gateway === "string" && gateway.includes(".")); - expect(privateListenHost, "OpenShell Docker network has no IPv4 gateway").toBeDefined(); - const recorderListenHost = privateListenHost as string; - const routedProxy = createToolDisclosureRecordingProxy({ - upstreamBaseUrl: hosted.endpointUrl, - allowRemoteHttpsUpstream: true, - listenHost: recorderListenHost, - allowAuthenticatedPrivateIpv4Listener: true, - requiredAuthorization: `Bearer ${routingIngressToken}`, - upstreamAuthorization: `Bearer ${hosted.apiKey}`, - requestTimeoutMs: ROUTED_PROXY_REQUEST_TIMEOUT_MS, - requestTransform: routedTransform.requestTransform, - }); - const routedProxyAddress = await routedProxy.start(); - cleanup.add("stop compositional routing recording proxy", () => routedProxy.stop()); - expect(routedProxyAddress.host).toBe(recorderListenHost); - expect(routedProxyAddress.port).toBeGreaterThanOrEqual(1_024); + expect(attestation.exitCode, resultText(attestation)).toBe(0); + } - const bridgeProbe = await host.command( - "docker", + for (const mode of MODES) { + const add = await host.nemoclaw( [ - "run", - "--rm", - "--pull=missing", - "--network", - OPENSHELL_DOCKER_NETWORK, - PRIVATE_BRIDGE_PROBE_IMAGE, - "nc", - "-zw5", - recorderListenHost, - String(routedProxyAddress.port), + sandboxName(mode), + "mcp", + "add", + MCP_SERVER_NAME, + "--url", + tunnel.mcpUrl, + "--env", + MCP_TOKEN_ENV, ], { - artifactName: "compositional-routing-private-reachability", - env: hostEnv(), - redactionValues: [routedProxyAddress.base_url], - timeoutMs: 30_000, + artifactName: `add-tool-disclosure-mcp-${mode}`, + env: hostEnv({ [MCP_TOKEN_ENV]: bearerToken }), + redactionValues: [bearerToken], + timeoutMs: 5 * 60_000, }, ); - expect( - bridgeProbe.exitCode, - `OpenShell Docker network cannot reach the authenticated routing bridge: ${resultText(bridgeProbe)}`, - ).toBe(0); + expect(add.exitCode, resultText(add)).toBe(0); + } - const configureInferenceRoute = async ( - baseUrl: string, - artifactSuffix: string, - credential: string, - timeoutMs: number, - ) => { - artifacts.addRedactionValues([baseUrl, credential]); - const providerEnv = hostEnv({ - [hosted.credentialEnv]: credential, + const results = []; + for (const mode of MODES) { + const runId = `performance-smoke-${AGENT}-${mode}-${TASK_ID}`; + const driver = buildAgentDriverCommand({ + openshellBin: process.env.OPENSHELL_BIN, + sandboxName: sandboxName(mode), + agent: AGENT, + prompt: frozenTask.prompt, + sessionId: runId, + }); + mcp.beginRun(runId); + const startedAt = process.hrtime.bigint(); + let invocation; + let calls; + try { + invocation = await host.command(driver.command, driver.args, { + artifactName: `invoke-tool-disclosure-${mode}`, + env: hostEnv(), + redactionValues: [...driver.redactions, hosted.apiKey, bearerToken], + timeoutMs: 10 * 60_000, }); - const update = await host.command( - "openshell", - [ - "provider", - "update", - hosted.providerName, - "--credential", - hosted.credentialEnv, - "--config", - `OPENAI_BASE_URL=${baseUrl}`, - ], - { - artifactName: `compositional-routing-provider-${artifactSuffix}`, - env: providerEnv, - redactionValues: [hosted.apiKey, routingIngressToken, baseUrl, credential], - timeoutMs: 2 * 60_000, - }, - ); - expect(update.exitCode, resultText(update)).toBe(0); - const select = await host.command( - "openshell", - [ - "inference", - "set", - "--no-verify", - "--provider", - hosted.providerName, - "--model", - hosted.model, - "--timeout", - String(timeoutMs / 1_000), - ], - { - artifactName: `compositional-routing-select-${artifactSuffix}`, - env: providerEnv, - redactionValues: [hosted.apiKey, routingIngressToken, baseUrl, credential], - timeoutMs: 2 * 60_000, - }, - ); - expect(select.exitCode, resultText(select)).toBe(0); - }; + } finally { + calls = mcp.endRun(); + } + const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); + const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); + const graded = gradeTaskRun(frozenTask, calls, finalOutput); + const outcome = invocation.timedOut + ? "timeout" + : invocation.exitCode === 0 + ? graded.outcome + : "model-error"; + results.push({ + run_id: runId, + mode, + outcome, + invocation: { + exit_code: invocation.exitCode, + timed_out: invocation.timedOut, + elapsed_ms: elapsedMs, + }, + synthetic_call_count: calls.length, + correctness: graded.correctness, + }); + } - let providerRestore: Promise | undefined; - const restoreInferenceRoute = (): Promise => - providerRestore ?? - (providerRestore = configureInferenceRoute( - hosted.endpointUrl, - "restore", - hosted.apiKey, - RESTORED_GATEWAY_REQUEST_TIMEOUT_MS, - ).catch((error: unknown) => { - providerRestore = undefined; - throw error; - })); - cleanup.add("restore hosted inference route", restoreInferenceRoute); + const routedRunId = `performance-smoke-${AGENT}-compositional-${TASK_ID}`; + const routedToolNames = new Set( + catalogPrefix.tools.map((tool) => `${MCP_SERVER_NAME}_${tool.definition.function.name}`), + ); + const expectedRoutedToolName = `${MCP_SERVER_NAME}_${frozenTask.expected_calls[0]?.tool_name ?? ""}`; + expect( + routedToolNames.has(expectedRoutedToolName), + "the routed replay target is not present in the reviewed catalog", + ).toBe(true); + const routedTransform = new CompositionalToolRoutingTransform({ + decomposer: createOpenAIChatTaskDecomposer({ + baseUrl: hosted.endpointUrl, + model: hosted.model, + apiKey: hosted.apiKey, + allowRemote: true, + reasoningControl: "enable_thinking_false", + jsonObjectResponse: true, + maxAttempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, + timeoutMs: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, + }), + embedder: new PortableHashingTextEmbedder(), + isRoutableTool: (name) => routedToolNames.has(name), + requireRouting: true, + }); + const networkInspect = await host.command( + "docker", + ["network", "inspect", "--format", "{{json .IPAM.Config}}", OPENSHELL_DOCKER_NETWORK], + { + artifactName: "compositional-routing-private-network", + env: hostEnv(), + timeoutMs: 30_000, + }, + ); + expect(networkInspect.exitCode, resultText(networkInspect)).toBe(0); + const networkIpam = JSON.parse(networkInspect.stdout) as Array<{ + Gateway?: unknown; + }>; + const privateListenHost = networkIpam + .map((entry) => entry.Gateway) + .find((gateway): gateway is string => typeof gateway === "string" && gateway.includes(".")); + expect(privateListenHost, "OpenShell Docker network has no IPv4 gateway").toBeDefined(); + const recorderListenHost = privateListenHost as string; + const routedProxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: hosted.endpointUrl, + allowRemoteHttpsUpstream: true, + listenHost: recorderListenHost, + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: `Bearer ${routingIngressToken}`, + upstreamAuthorization: `Bearer ${hosted.apiKey}`, + requestTimeoutMs: ROUTED_PROXY_REQUEST_TIMEOUT_MS, + requestTransform: routedTransform.requestTransform, + }); + const routedProxyAddress = await routedProxy.start(); + cleanup.add("stop compositional routing recording proxy", () => routedProxy.stop()); + expect(routedProxyAddress.host).toBe(recorderListenHost); + expect(routedProxyAddress.port).toBeGreaterThanOrEqual(1_024); - let routedReplay: Record | undefined; + const bridgeProbe = await host.command( + "docker", + [ + "run", + "--rm", + "--pull=missing", + "--network", + OPENSHELL_DOCKER_NETWORK, + PRIVATE_BRIDGE_PROBE_IMAGE, + "nc", + "-zw5", + recorderListenHost, + String(routedProxyAddress.port), + ], + { + artifactName: "compositional-routing-private-reachability", + env: hostEnv(), + redactionValues: [routedProxyAddress.base_url], + timeoutMs: 30_000, + }, + ); + expect( + bridgeProbe.exitCode, + `OpenShell Docker network cannot reach the authenticated routing bridge: ${resultText(bridgeProbe)}`, + ).toBe(0); + + const configureInferenceRoute = async ( + baseUrl: string, + artifactSuffix: string, + credential: string, + timeoutMs: number, + ) => { + artifacts.addRedactionValues([baseUrl, credential]); + const providerEnv = hostEnv({ + [hosted.credentialEnv]: credential, + }); + const update = await host.command( + "openshell", + [ + "provider", + "update", + hosted.providerName, + "--credential", + hosted.credentialEnv, + "--config", + `OPENAI_BASE_URL=${baseUrl}`, + ], + { + artifactName: `compositional-routing-provider-${artifactSuffix}`, + env: providerEnv, + redactionValues: [hosted.apiKey, routingIngressToken, baseUrl, credential], + timeoutMs: 2 * 60_000, + }, + ); + expect(update.exitCode, resultText(update)).toBe(0); + const select = await host.command( + "openshell", + [ + "inference", + "set", + "--no-verify", + "--provider", + hosted.providerName, + "--model", + hosted.model, + "--timeout", + String(timeoutMs / 1_000), + ], + { + artifactName: `compositional-routing-select-${artifactSuffix}`, + env: providerEnv, + redactionValues: [hosted.apiKey, routingIngressToken, baseUrl, credential], + timeoutMs: 2 * 60_000, + }, + ); + expect(select.exitCode, resultText(select)).toBe(0); + }; + + let providerRestore: Promise | undefined; + const restoreInferenceRoute = (): Promise => + providerRestore ?? + (providerRestore = configureInferenceRoute( + hosted.endpointUrl, + "restore", + hosted.apiKey, + RESTORED_GATEWAY_REQUEST_TIMEOUT_MS, + ).catch((error: unknown) => { + providerRestore = undefined; + throw error; + })); + cleanup.add("restore hosted inference route", restoreInferenceRoute); + + let routedReplay: Record | undefined; + try { + await configureInferenceRoute( + `${routedProxyAddress.base_url}/v1`, + "enable", + routingIngressToken, + ROUTED_GATEWAY_REQUEST_TIMEOUT_MS, + ); + const driver = buildAgentDriverCommand({ + openshellBin: process.env.OPENSHELL_BIN, + sandboxName: sandboxName("direct"), + agent: AGENT, + prompt: frozenTask.prompt, + sessionId: routedRunId, + }); + mcp.beginRun(routedRunId); + routedProxy.beginRun(routedRunId); + const startedAt = process.hrtime.bigint(); + let invocation; + let calls; + let recorderEvents; try { - await configureInferenceRoute( - `${routedProxyAddress.base_url}/v1`, - "enable", - routingIngressToken, - ROUTED_GATEWAY_REQUEST_TIMEOUT_MS, - ); - const driver = buildAgentDriverCommand({ - openshellBin: process.env.OPENSHELL_BIN, - sandboxName: sandboxName("direct"), - agent: AGENT, - prompt: frozenTask.prompt, - sessionId: routedRunId, - }); - mcp.beginRun(routedRunId); - routedProxy.beginRun(routedRunId); - const startedAt = process.hrtime.bigint(); - let invocation; - let calls; - let recorderEvents; - try { - invocation = await host.command(driver.command, driver.args, { - artifactName: "invoke-tool-disclosure-compositional", - env: hostEnv(), - redactionValues: [ - ...driver.redactions, - hosted.apiKey, - bearerToken, - routingIngressToken, - routedProxyAddress.base_url, - ], - timeoutMs: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, - }); - } finally { - calls = mcp.endRun(); - recorderEvents = routedProxy.endRun(); - } - const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); - const routeEvidence = await routedTransform.consumeEvidence(routedRunId); - const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); - const graded = gradeTaskRun(frozenTask, calls, finalOutput); - const outcome = invocation.timedOut - ? "timeout" - : invocation.exitCode === 0 - ? graded.outcome - : "model-error"; - routedReplay = { - schema_version: "nemoclaw.compositional_tool_routing_agent_smoke.v1", - generated_at: new Date().toISOString(), - claim_eligible: false, - run_id: routedRunId, - outcome, - invocation: { - exit_code: invocation.exitCode, - timed_out: invocation.timedOut, - elapsed_ms: elapsedMs, - }, - synthetic_call_count: calls.length, - correctness: graded.correctness, - expected_routed_tool_name: expectedRoutedToolName, - routing_configuration: { - reasoning_control: "enable_thinking_false", - output_mode: "json-object", - max_attempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, - timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, - route_only_run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, - proxy_request_timeout_ms: ROUTED_PROXY_REQUEST_TIMEOUT_MS, - gateway_request_timeout_ms: ROUTED_GATEWAY_REQUEST_TIMEOUT_MS, - agent_invocation_timeout_ms: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, - baseline_gateway_request_timeout_ms: RESTORED_GATEWAY_REQUEST_TIMEOUT_MS, - gateway_transport: "authenticated-private-host-bridge", - embedding: "portable-lexical-hashing", - }, - route_evidence: routeEvidence, - recorder_events: recorderEvents, - limitations: [ - "This single routed replay is an end-to-end wiring check, not a performance or quality claim.", - "The portable lexical embedder is used only for an ordinary-runner smoke path.", - "The replay uses an authenticated private runner bridge and replaces its ephemeral ingress credential before the hosted request.", - "The bridge uses host-local HTTP inside the trusted runner and Docker-network boundary.", - "Proxy and gateway timeouts apply per request; the routed agent invocation has its own overall bound.", - "The routed replay is separate from the frozen direct/progressive comparison.", + invocation = await host.command(driver.command, driver.args, { + artifactName: "invoke-tool-disclosure-compositional", + env: hostEnv(), + redactionValues: [ + ...driver.redactions, + hosted.apiKey, + bearerToken, + routingIngressToken, + routedProxyAddress.base_url, ], - }; + timeoutMs: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, + }); } finally { - await restoreInferenceRoute(); + calls = mcp.endRun(); + recorderEvents = routedProxy.endRun(); } - - await artifacts.writeJson("compositional-routing-agent-smoke.json", routedReplay); - expect(routedReplay).toBeDefined(); - const replay = routedReplay as { - outcome: string; - correctness: { task_success: boolean }; - synthetic_call_count: number; - route_evidence: Awaited>; - recorder_events: Array<{ - model_call_sequence: number | null; - visible_tool_count: number; - }>; - }; - expect(replay.outcome).toBe("success"); - expect(replay.correctness.task_success).toBe(true); - expect(replay.synthetic_call_count).toBe(1); - expect(replay.route_evidence.length).toBeGreaterThan(0); - const [firstRoute] = replay.route_evidence; - expect(firstRoute.routing.fallback).toBeNull(); - expect(firstRoute.transform_bypass).toBeNull(); - expect(firstRoute.routable_tool_count).toBe(CATALOG_SIZE); - expect(firstRoute.forwarded_tool_count).toBeLessThan(firstRoute.source_tool_count); - expect(firstRoute.routing.selected_tool_names).toContain(expectedRoutedToolName); - const firstModelEvent = replay.recorder_events.find((event) => event.model_call_sequence === 1); - expect(firstModelEvent?.visible_tool_count).toBe(firstRoute.forwarded_tool_count); - - await artifacts.writeJson("tool-disclosure-performance-smoke.json", { - schema_version: "nemoclaw.tool_disclosure_performance_smoke.v1", + const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); + const routeEvidence = await routedTransform.consumeEvidence(routedRunId); + const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); + const graded = gradeTaskRun(frozenTask, calls, finalOutput); + const outcome = invocation.timedOut + ? "timeout" + : invocation.exitCode === 0 + ? graded.outcome + : "model-error"; + routedReplay = { + schema_version: "nemoclaw.compositional_tool_routing_agent_smoke.v1", generated_at: new Date().toISOString(), claim_eligible: false, - sut_git_sha: gitSha(), - profile: { - agent: AGENT, - modes: MODES, - catalog_seed: catalog.seed, - catalog_size: CATALOG_SIZE, - catalog_tools_sha256: catalogPrefix.tools_sha256, - task_id: frozenTask.id, - task_kind: frozenTask.kind, - task_set_sha256: primaryTasks.tasks_sha256, - repetitions_per_mode: 1, - observed_runs: results.length, + run_id: routedRunId, + outcome, + invocation: { + exit_code: invocation.exitCode, + timed_out: invocation.timedOut, + elapsed_ms: elapsedMs, }, - inference: { - api: "openai-compatible", - model_id: hosted.model, + synthetic_call_count: calls.length, + correctness: graded.correctness, + expected_routed_tool_name: expectedRoutedToolName, + routing_configuration: { + reasoning_control: "enable_thinking_false", + output_mode: "json-object", + max_attempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, + timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, + route_only_run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, + proxy_request_timeout_ms: ROUTED_PROXY_REQUEST_TIMEOUT_MS, + gateway_request_timeout_ms: ROUTED_GATEWAY_REQUEST_TIMEOUT_MS, + agent_invocation_timeout_ms: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, + baseline_gateway_request_timeout_ms: RESTORED_GATEWAY_REQUEST_TIMEOUT_MS, + gateway_transport: "authenticated-private-host-bridge", + embedding: "portable-lexical-hashing", }, - results, + route_evidence: routeEvidence, + recorder_events: recorderEvents, limitations: [ - "This performance smoke test verifies live wiring and task completion; it is not the complete two-campaign performance test.", - "One observation per mode is insufficient for performance or quality claims.", - "The fixed progressive-then-direct order makes elapsed times informational and subject to cold-cache and ordering effects.", - "The performance smoke test does not collect vLLM tokenizer, token-counter, or request-recorder evidence.", + "This single routed replay is an end-to-end wiring check, not a performance or quality claim.", + "The portable lexical embedder is used only for an ordinary-runner smoke path.", + "The replay uses an authenticated private runner bridge and replaces its ephemeral ingress credential before the hosted request.", + "The bridge uses host-local HTTP inside the trusted runner and Docker-network boundary.", + "Proxy and gateway timeouts apply per request; the routed agent invocation has its own overall bound.", + "The routed replay is separate from the frozen direct/progressive comparison.", ], - }); + }; + } finally { + await restoreInferenceRoute(); + } - expect(results).toHaveLength(2); - for (const result of results) { - expect(result.outcome, `${result.mode} performance smoke outcome`).toBe("success"); - expect(result.correctness.task_success, `${result.mode} task correctness`).toBe(true); - expect(result.synthetic_call_count, `${result.mode} synthetic call count`).toBe(1); - } - }, -); + await artifacts.writeJson("compositional-routing-agent-smoke.json", routedReplay); + expect(routedReplay).toBeDefined(); + const replay = routedReplay as { + outcome: string; + correctness: { task_success: boolean }; + synthetic_call_count: number; + route_evidence: Awaited>; + recorder_events: Array<{ + model_call_sequence: number | null; + visible_tool_count: number; + }>; + }; + expect(replay.outcome).toBe("success"); + expect(replay.correctness.task_success).toBe(true); + expect(replay.synthetic_call_count).toBe(1); + expect(replay.route_evidence.length).toBeGreaterThan(0); + const [firstRoute] = replay.route_evidence; + expect(firstRoute.routing.fallback).toBeNull(); + expect(firstRoute.transform_bypass).toBeNull(); + expect(firstRoute.routable_tool_count).toBe(CATALOG_SIZE); + expect(firstRoute.forwarded_tool_count).toBeLessThan(firstRoute.source_tool_count); + expect(firstRoute.routing.selected_tool_names).toContain(expectedRoutedToolName); + const firstModelEvent = replay.recorder_events.find((event) => event.model_call_sequence === 1); + expect(firstModelEvent?.visible_tool_count).toBe(firstRoute.forwarded_tool_count); + + await artifacts.writeJson("tool-disclosure-performance-smoke.json", { + schema_version: "nemoclaw.tool_disclosure_performance_smoke.v1", + generated_at: new Date().toISOString(), + claim_eligible: false, + sut_git_sha: gitSha(), + profile: { + agent: AGENT, + modes: MODES, + catalog_seed: catalog.seed, + catalog_size: CATALOG_SIZE, + catalog_tools_sha256: catalogPrefix.tools_sha256, + task_id: frozenTask.id, + task_kind: frozenTask.kind, + task_set_sha256: primaryTasks.tasks_sha256, + repetitions_per_mode: 1, + observed_runs: results.length, + }, + inference: { + api: "openai-compatible", + model_id: hosted.model, + }, + results, + limitations: [ + "This performance smoke test verifies live wiring and task completion; it is not the complete two-campaign performance test.", + "One observation per mode is insufficient for performance or quality claims.", + "The fixed progressive-then-direct order makes elapsed times informational and subject to cold-cache and ordering effects.", + "The performance smoke test does not collect vLLM tokenizer, token-counter, or request-recorder evidence.", + ], + }); + + expect(results).toHaveLength(2); + for (const result of results) { + expect(result.outcome, `${result.mode} performance smoke outcome`).toBe("success"); + expect(result.correctness.task_success, `${result.mode} task correctness`).toBe(true); + expect(result.synthetic_call_count, `${result.mode} synthetic call count`).toBe(1); + } +}); From c5f9ee6c738a3d861cce12e6fe7abefc2ae2fd54 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 12:36:57 -0700 Subject: [PATCH 25/31] ci: run performance smoke on Brev Signed-off-by: Aaron Erickson --- .github/workflows/e2e-branch-validation.yaml | 48 +++++++++++++++++++- test/brev-nightly-workflow.test.ts | 32 ++++++++++++- test/brev-remote-vitest.test.ts | 22 +++++++++ test/e2e/brev-e2e.test.ts | 48 +++++++++++++++++++- tools/e2e/brev-remote-vitest.mts | 22 +++++++++ 5 files changed, 168 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-branch-validation.yaml b/.github/workflows/e2e-branch-validation.yaml index d80d1ee877..95765466a3 100644 --- a/.github/workflows/e2e-branch-validation.yaml +++ b/.github/workflows/e2e-branch-validation.yaml @@ -46,6 +46,9 @@ name: E2E / Branch Validation # OpenAI-compatible endpoint. Creates its own sandbox # (e2e-msg-compat) on a separate fresh instance. # dashboard-remote-bind — Verifies opt-in remote dashboard forwards bind 0.0.0.0. +# tool-disclosure-performance-smoke — Runs the claim-ineligible hosted-inference +# performance smoke on an ephemeral Brev CPU VM and +# retains its sanitized evidence. (~60 min) # gpu — Provisions a Brev GPU VM and runs the Ollama GPU E2E # sandbox proof suite from source. (~45 min) # all — Runs credential-sanitization + telegram-injection, each with @@ -73,6 +76,7 @@ on: - messaging-providers - messaging-compatible-endpoint - dashboard-remote-bind + - tool-disclosure-performance-smoke - gpu - all use_launchable: @@ -188,7 +192,7 @@ jobs: TEST_SUITE: ${{ inputs.test_suite }} run: | case "$TEST_SUITE" in - full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all) ;; + full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|tool-disclosure-performance-smoke|gpu|all) ;; *) echo "::error::test_suite is not one of the supported Brev E2E suites" exit 1 @@ -213,7 +217,7 @@ jobs: - name: Checkout target branch uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - ref: ${{ env.RESOLVED_BRANCH || inputs.branch || 'main' }} + ref: ${{ env.RESOLVED_BRANCH || inputs.branch || github.ref }} persist-credentials: false - id: tested-ref @@ -286,6 +290,46 @@ jobs: KEEP_ALIVE: ${{ inputs.keep_alive }} run: npx vitest run --project e2e-branch-validation --silent=false --reporter=default + - name: Collect Brev tool-disclosure performance smoke artifacts + if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke' + env: + INSTANCE: ${{ env.BREV_E2E_INSTANCE_NAME }} + run: | + set -euo pipefail + artifact_dir="brev-tool-disclosure-performance-smoke-artifacts" + remote_dir="/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts" + mkdir -p "$artifact_dir" + brev refresh >/dev/null + ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR \ + -o ConnectTimeout=10 "$INSTANCE" \ + "test -d '$remote_dir' && tar -C '$remote_dir' -czf - ." \ + | tar -C "$artifact_dir" -xzf - + test -f "$artifact_dir/tool-disclosure-hosted-inference-performance-smoke-completes-one-frozen-task-in-both-modes/tool-disclosure-performance-smoke.json" + + - name: Scan Brev tool-disclosure performance smoke artifacts + if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke' + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + artifact_dir="brev-tool-disclosure-performance-smoke-artifacts" + test -d "$artifact_dir" + if [ -n "${NVIDIA_INFERENCE_API_KEY:-}" ] && \ + grep -R -I -l -F -- "$NVIDIA_INFERENCE_API_KEY" "$artifact_dir" >/dev/null; then + echo "::error::Brev performance smoke artifacts contain an unredacted inference credential" + exit 1 + fi + + - name: Upload Brev tool-disclosure performance smoke artifacts + if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-brev-tool-disclosure-performance-smoke-${{ github.run_attempt }} + path: brev-tool-disclosure-performance-smoke-artifacts/ + if-no-files-found: error + include-hidden-files: false + retention-days: 14 + # Collect debugging artifacts from the Brev VM on failure before the # instance gets torn down. Captures the onboard log, sandbox list, # docker state, and gateway status so downstream onboard failures are diff --git a/test/brev-nightly-workflow.test.ts b/test/brev-nightly-workflow.test.ts index 7cc0693178..d1900631a1 100644 --- a/test/brev-nightly-workflow.test.ts +++ b/test/brev-nightly-workflow.test.ts @@ -96,6 +96,7 @@ describe("Brev nightly workflow contract", () => { "pull-requests": "read", }); expect(checkout?.with?.["persist-credentials"]).toBe(false); + expect(checkout?.with?.ref).toBe("${{ env.RESOLVED_BRANCH || inputs.branch || github.ref }}"); expect(resolveBranch?.env?.PR_NUMBER).toBe("${{ inputs.pr_number }}"); expect(resolveBranch?.run).not.toContain("gh pr view ${{"); expect(validation?.outputs?.tested_sha).toBe("${{ steps.tested-ref.outputs.sha }}"); @@ -129,7 +130,7 @@ describe("Brev nightly workflow contract", () => { expect(validation?.env?.TEST_SUITE).toBe("${{ inputs.test_suite }}"); expect(validation?.run).toContain( - "full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all", + "full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|tool-disclosure-performance-smoke|gpu|all", ); expect(validation?.run).toContain("exit 1"); expect(steps.indexOf(validation as NonNullable)).toBeLessThan( @@ -137,6 +138,35 @@ describe("Brev nightly workflow contract", () => { ); }); + it("collects and scans Brev performance smoke evidence before deleting the VM", () => { + const steps = branchValidation.jobs?.["e2e-branch-validation"]?.steps ?? []; + const collect = steps.find( + (step) => step.name === "Collect Brev tool-disclosure performance smoke artifacts", + ); + const scan = steps.find( + (step) => step.name === "Scan Brev tool-disclosure performance smoke artifacts", + ); + const upload = steps.find( + (step) => step.name === "Upload Brev tool-disclosure performance smoke artifacts", + ); + const cleanup = steps.find((step) => step.name === "Delete Brev instance"); + + expect(collect?.if).toContain("tool-disclosure-performance-smoke"); + expect(collect?.run).toContain("/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"); + expect(collect?.run).toContain("tar -C"); + expect(scan?.env?.NVIDIA_INFERENCE_API_KEY).toBe("${{ secrets.NVIDIA_INFERENCE_API_KEY }}"); + expect(scan?.run).toContain("grep -R -I -l -F"); + expect(upload?.uses).toBe("actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"); + expect(upload?.with).toMatchObject({ + name: "e2e-brev-tool-disclosure-performance-smoke-${{ github.run_attempt }}", + "if-no-files-found": "error", + "retention-days": 14, + }); + expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( + steps.indexOf(upload as NonNullable), + ); + }); + it("runs stateful messaging targets on separate fresh instances", () => { expect(nightly.jobs?.["brev-nightly-e2e"]?.strategy?.matrix?.test_suite).toEqual([ "all", diff --git a/test/brev-remote-vitest.test.ts b/test/brev-remote-vitest.test.ts index 988429d8a6..62aaabddfe 100644 --- a/test/brev-remote-vitest.test.ts +++ b/test/brev-remote-vitest.test.ts @@ -9,14 +9,19 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { + BREV_CLOUDFLARED_DEB_SHA256, + BREV_CLOUDFLARED_VERSION, BREV_MESSAGING_COMPAT_TIMEOUT_MS, BREV_MESSAGING_PROVIDER_TIMEOUT_MS, BREV_REMOTE_WRAPPER_GRACE_MS, BREV_SECURITY_SUITE_TIMEOUT_MS, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS, BREV_WORKFLOW_OWNERSHIP_ENV, brevSuiteHarnessSandboxName, brevSuiteNeedsHarnessSandbox, brevWorkflowOwnsInstance, + buildBrevCloudflaredInstallCommands, buildBrevRemoteVitestCommand, } from "../tools/e2e/brev-remote-vitest.mts"; @@ -98,6 +103,7 @@ describe("Brev remote Vitest command", () => { expect(BREV_SECURITY_SUITE_TIMEOUT_MS).toBe(20 * 60_000); expect(BREV_MESSAGING_PROVIDER_TIMEOUT_MS).toBe(70 * 60_000); expect(BREV_MESSAGING_COMPAT_TIMEOUT_MS).toBe(40 * 60_000); + expect(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS).toBe(55 * 60_000); expect(BREV_REMOTE_WRAPPER_GRACE_MS).toBe(120_000); }); @@ -114,9 +120,25 @@ describe("Brev remote Vitest command", () => { expect(brevSuiteNeedsHarnessSandbox("gpu")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("messaging-compatible-endpoint")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("messaging-providers")).toBe(false); + expect(brevSuiteNeedsHarnessSandbox(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE)).toBe(false); expect(brevSuiteHarnessSandboxName("all")).toBeUndefined(); expect(brevSuiteHarnessSandboxName("messaging-compatible-endpoint")).toBeUndefined(); expect(brevSuiteHarnessSandboxName("messaging-providers")).toBeUndefined(); + expect( + brevSuiteHarnessSandboxName(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE), + ).toBeUndefined(); + }); + + it("installs the smoke tunnel prerequisite from an immutable reviewed package", () => { + const command = buildBrevCloudflaredInstallCommands().join("\n"); + + expect(command).toContain(BREV_CLOUDFLARED_VERSION); + expect(command).toContain(BREV_CLOUDFLARED_DEB_SHA256); + expect(command).toContain("sha256sum -c -"); + expect(command).toContain("dpkg-deb -f"); + expect(command).toContain("sudo dpkg -i"); + expect(command).not.toContain("apt-get install"); + expect(command).not.toContain("pkg.cloudflare.com"); }); it("preserves harness onboarding for single-target suites", () => { diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index b40f3c7247..fc427d27bc 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -28,7 +28,8 @@ * Optional env vars: * TEST_SUITE — which test to run: full (default), deploy-cli, gpu, * credential-sanitization, telegram-injection, messaging-providers, - * messaging-compatible-endpoint, dashboard-remote-bind, all + * messaging-compatible-endpoint, dashboard-remote-bind, + * tool-disclosure-performance-smoke, all * BREV_MIN_VCPU — Minimum vCPUs for CPU instance (default: 4) * BREV_MIN_RAM — Minimum RAM in GB for CPU instance (default: 16) * BREV_PROVIDER — Cloud provider filter for brev search (default: gcp for CPU, any for GPU) @@ -58,9 +59,13 @@ import { BREV_MESSAGING_PROVIDER_TIMEOUT_MS, BREV_REMOTE_WRAPPER_GRACE_MS, BREV_SECURITY_SUITE_TIMEOUT_MS, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS, brevSuiteHarnessSandboxName, brevSuiteNeedsHarnessSandbox, brevWorkflowOwnsInstance, + buildBrevCloudflaredInstallCommands, buildBrevRemoteVitestCommand, } from "../../tools/e2e/brev-remote-vitest.mts"; @@ -76,6 +81,8 @@ const TEST_SUITE = process.env.TEST_SUITE || "full"; const REPO_DIR = path.resolve(import.meta.dirname, "../.."); const CLI_PATH = path.join(REPO_DIR, "bin", "nemoclaw.js"); const GPU_TEST_SUITE = TEST_SUITE === "gpu"; +const TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE = + TEST_SUITE === BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE; const BREV_PROVIDER = process.env.BREV_PROVIDER || (GPU_TEST_SUITE ? "" : "gcp"); const BREV_CREATE_TIMEOUT_SECONDS = parseInt( process.env.BREV_CREATE_TIMEOUT_SECONDS || (GPU_TEST_SUITE ? "1200" : "180"), @@ -288,6 +295,19 @@ function sshEnv( // sandbox creation while auto-loading a very large default Ollama model. envParts.push(`export NEMOCLAW_MODEL='${shellEscape(gpuE2eModel)}'`); } + if (TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE) { + const testedSha = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: REPO_DIR, + encoding: "utf8", + }).trim(); + if (!/^[a-f0-9]{40}$/u.test(testedSha)) { + throw new Error("Brev performance smoke requires an exact local git SHA"); + } + envParts.push( + `export E2E_ARTIFACT_DIR='${shellEscape(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR)}'`, + `export GITHUB_SHA='${testedSha}'`, + ); + } // Forward optional messaging tokens for the messaging-providers test for (const key of [ "TELEGRAM_BOT_TOKEN", @@ -766,6 +786,15 @@ function prepareGpuDockerRuntime(elapsed: () => string): void { console.log(`[${elapsed()}] NVIDIA Docker runtime ready`); } +function prepareToolDisclosurePerformanceSmoke(elapsed: () => string): void { + console.log(`[${elapsed()}] Installing verified cloudflared prerequisite...`); + ssh(buildBrevCloudflaredInstallCommands().join(" && "), { + timeout: 180_000, + stream: true, + }); + console.log(`[${elapsed()}] Verified cloudflared prerequisite ready`); +} + /** * Bootstrap the launchable environment on the remote VM: * rsync branch code, install deps, build plugin, and npm link the CLI. @@ -1125,6 +1154,10 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { const result = bootstrapLaunchable(elapsed); remoteDir = result.remoteDir; + if (TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE) { + prepareToolDisclosurePerformanceSmoke(elapsed); + } + if (result.needsOnboard) { pollForSandboxReady(elapsed); writeManualRegistry(elapsed); @@ -1188,6 +1221,19 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { 1_800_000, ); + it.runIf(TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE)( + "tool-disclosure performance smoke passes on Brev CPU VM", + () => { + const output = runRemoteVitest( + "e2e-live", + "test/e2e/live/tool-disclosure-performance-smoke.test.ts", + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS, + ); + expectVitestPassed(output); + }, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS + BREV_REMOTE_WRAPPER_GRACE_MS, + ); + it.runIf(TEST_SUITE === "credential-sanitization" || TEST_SUITE === "all")( "credential sanitization suite passes on remote VM", () => { diff --git a/tools/e2e/brev-remote-vitest.mts b/tools/e2e/brev-remote-vitest.mts index 19b8c30851..82994695a3 100644 --- a/tools/e2e/brev-remote-vitest.mts +++ b/tools/e2e/brev-remote-vitest.mts @@ -8,8 +8,15 @@ export type BrevVitestProject = "cli" | "e2e-live"; export const BREV_SECURITY_SUITE_TIMEOUT_MS = 20 * 60_000; export const BREV_MESSAGING_PROVIDER_TIMEOUT_MS = 70 * 60_000; export const BREV_MESSAGING_COMPAT_TIMEOUT_MS = 40 * 60_000; +export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS = 55 * 60_000; export const BREV_REMOTE_WRAPPER_GRACE_MS = 120_000; export const BREV_WORKFLOW_OWNERSHIP_ENV = "NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE"; +export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE = "tool-disclosure-performance-smoke"; +export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR = + "/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"; +export const BREV_CLOUDFLARED_VERSION = "2026.6.1"; +export const BREV_CLOUDFLARED_DEB_SHA256 = + "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"; const BREV_SUITES_WITHOUT_HARNESS_SANDBOX = new Set([ "all", @@ -17,6 +24,7 @@ const BREV_SUITES_WITHOUT_HARNESS_SANDBOX = new Set([ "gpu", "messaging-compatible-endpoint", "messaging-providers", + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE, ]); export function brevSuiteNeedsHarnessSandbox(testSuite: string): boolean { @@ -54,3 +62,17 @@ export function buildBrevRemoteVitestCommand(project: BrevVitestProject, target: `NEMOCLAW_RUN_LIVE_E2E=1 ${vitestCommand}`, ].join(" && "); } + +export function buildBrevCloudflaredInstallCommands(): string[] { + return [ + "set -euo pipefail", + `cloudflared_deb=/tmp/cloudflared-${BREV_CLOUDFLARED_VERSION}-linux-amd64.deb`, + `curl -fL https://github.com/cloudflare/cloudflared/releases/download/${BREV_CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb -o \"$cloudflared_deb\"`, + `printf '%s %s\\n' ${BREV_CLOUDFLARED_DEB_SHA256} \"$cloudflared_deb\" | sha256sum -c -`, + `test \"$(dpkg-deb -f \"$cloudflared_deb\" Package)\" = cloudflared`, + `test \"$(dpkg-deb -f \"$cloudflared_deb\" Version)\" = ${BREV_CLOUDFLARED_VERSION}`, + `test \"$(dpkg-deb -f \"$cloudflared_deb\" Architecture)\" = amd64`, + `sudo dpkg -i \"$cloudflared_deb\"`, + `cloudflared --version | grep -F 'cloudflared version ${BREV_CLOUDFLARED_VERSION}'`, + ]; +} From 3e07658e5d382e64763e3bc23a952132ad869875 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 12:54:42 -0700 Subject: [PATCH 26/31] ci: relay Brev performance smoke tunnel Signed-off-by: Aaron Erickson --- .github/workflows/e2e-branch-validation.yaml | 20 ++ .../tool-disclosure/quick-tunnel.ts | 32 +++- .../tool-disclosure/smoke-mcp-transport.ts | 72 +++++++ test/brev-nightly-workflow.test.ts | 10 + test/brev-remote-vitest.test.ts | 59 ++++-- test/e2e/brev-e2e.test.ts | 175 ++++++++++++++++-- .../tool-disclosure-performance-smoke.test.ts | 35 +++- .../tool-disclosure-quick-tunnel.test.ts | 38 ++++ ...ool-disclosure-smoke-mcp-transport.test.ts | 69 +++++++ tools/e2e/brev-remote-vitest.mts | 59 ++++-- 10 files changed, 512 insertions(+), 57 deletions(-) create mode 100644 scripts/performance/tool-disclosure/smoke-mcp-transport.ts create mode 100644 test/performance/tool-disclosure-smoke-mcp-transport.test.ts diff --git a/.github/workflows/e2e-branch-validation.yaml b/.github/workflows/e2e-branch-validation.yaml index 95765466a3..d4088f65e0 100644 --- a/.github/workflows/e2e-branch-validation.yaml +++ b/.github/workflows/e2e-branch-validation.yaml @@ -271,6 +271,26 @@ jobs: - name: Build CLI run: npm run build:cli + - name: Install and verify Brev performance smoke tunnel prerequisite + if: inputs.test_suite == 'tool-disclosure-performance-smoke' + env: + CLOUDFLARED_VERSION: "2026.6.1" + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" + run: | + set -euo pipefail + cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" + curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" + printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - + package="$(dpkg-deb -f "${cloudflared_deb}" Package)" + version="$(dpkg-deb -f "${cloudflared_deb}" Version)" + architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" + if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then + printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 + exit 1 + fi + sudo dpkg -i "${cloudflared_deb}" + cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" + - name: Run ephemeral Brev E2E env: NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1" diff --git a/scripts/performance/tool-disclosure/quick-tunnel.ts b/scripts/performance/tool-disclosure/quick-tunnel.ts index 4d8836b932..5f07bf373e 100644 --- a/scripts/performance/tool-disclosure/quick-tunnel.ts +++ b/scripts/performance/tool-disclosure/quick-tunnel.ts @@ -90,6 +90,22 @@ function stopChild(child: ChildProcess): Promise { }); } +function readLogTail(file: string, maxBytes = 8_192): string { + let descriptor: number | undefined; + try { + descriptor = fs.openSync(file, "r"); + const size = fs.fstatSync(descriptor).size; + const length = Math.min(size, maxBytes); + const buffer = Buffer.alloc(length); + fs.readSync(descriptor, buffer, 0, length, Math.max(0, size - length)); + return buffer.toString("utf8"); + } catch { + return ""; + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + async function probe(origin: string, fetchImpl: typeof fetch): Promise { try { const response = await fetchImpl(`${origin}/mcp`, { @@ -113,36 +129,34 @@ export async function startQuickTunnel(options: { }): Promise { const args = buildQuickTunnelArgs(options.port); const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-")); + const logFile = path.join(isolatedHome, "cloudflared.log"); const removeIsolatedHome = (): void => fs.rmSync(isolatedHome, { recursive: true, force: true }); let child: ChildProcess; + let logDescriptor: number | undefined; try { + logDescriptor = fs.openSync(logFile, "a"); child = spawn(options.binary ?? "cloudflared", args, { detached: false, env: buildQuickTunnelEnvironment(options.env ?? process.env, isolatedHome), - stdio: ["ignore", "pipe", "pipe"], + stdio: ["ignore", logDescriptor, logDescriptor], }); } catch (error) { + if (logDescriptor !== undefined) fs.closeSync(logDescriptor); removeIsolatedHome(); throw error; } + if (logDescriptor !== undefined) fs.closeSync(logDescriptor); const timeoutMs = options.timeoutMs ?? 45_000; const fetchImpl = options.fetchImpl ?? fetch; let origin: string | null = null; - let carry = ""; let spawnError: Error | undefined; - const inspect = (chunk: Buffer | string): void => { - const candidate = `${carry}${chunk.toString()}`; - origin = parseQuickTunnelOrigin(candidate) ?? origin; - carry = candidate.slice(-512); - }; - child.stdout?.on("data", inspect); - child.stderr?.on("data", inspect); child.once("error", (error) => { spawnError = error; }); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { + origin = parseQuickTunnelOrigin(readLogTail(logFile)) ?? origin; if (spawnError) break; if (child.exitCode !== null || child.signalCode !== null) break; if (origin && (await probe(origin, fetchImpl))) { diff --git a/scripts/performance/tool-disclosure/smoke-mcp-transport.ts b/scripts/performance/tool-disclosure/smoke-mcp-transport.ts new file mode 100644 index 0000000000..078070ba08 --- /dev/null +++ b/scripts/performance/tool-disclosure/smoke-mcp-transport.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const PERFORMANCE_SMOKE_MCP_URL_ENV = "NEMOCLAW_TOOL_DISCLOSURE_PERFORMANCE_MCP_URL"; +export const PERFORMANCE_SMOKE_MCP_PORT_ENV = "NEMOCLAW_TOOL_DISCLOSURE_PERFORMANCE_MCP_PORT"; + +export type PerformanceSmokeMcpTransport = + | { kind: "local-quick-tunnel" } + | { kind: "external-ssh-forward"; mcpUrl: string; listenPort: number }; + +function externalMcpUrl(value: string): string { + const url = new URL(value); + if ( + url.protocol !== "https:" || + url.username !== "" || + url.password !== "" || + url.port !== "" || + url.search !== "" || + url.hash !== "" || + url.pathname !== "/mcp" || + !/^[a-z0-9-]+\.trycloudflare\.com$/u.test(url.hostname) + ) { + throw new Error("external performance smoke MCP URL must be an exact trycloudflare /mcp URL"); + } + return url.toString(); +} + +export function resolvePerformanceSmokeMcpTransport( + env: NodeJS.ProcessEnv = process.env, +): PerformanceSmokeMcpTransport { + const rawUrl = env[PERFORMANCE_SMOKE_MCP_URL_ENV]?.trim() ?? ""; + const rawPort = env[PERFORMANCE_SMOKE_MCP_PORT_ENV]?.trim() ?? ""; + if (!rawUrl && !rawPort) return { kind: "local-quick-tunnel" }; + if (!rawUrl || !rawPort) { + throw new Error("external performance smoke MCP URL and port must be configured together"); + } + if (!/^[0-9]{1,5}$/u.test(rawPort)) { + throw new Error("external performance smoke MCP port must be an integer"); + } + const listenPort = Number(rawPort); + if (listenPort < 1_024 || listenPort > 65_535) { + throw new Error("external performance smoke MCP port must be between 1024 and 65535"); + } + return { + kind: "external-ssh-forward", + mcpUrl: externalMcpUrl(rawUrl), + listenPort, + }; +} + +export async function waitForPerformanceSmokeMcpEndpoint( + mcpUrl: string, + options: { fetchImpl?: typeof fetch; timeoutMs?: number } = {}, +): Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const deadline = Date.now() + (options.timeoutMs ?? 30_000); + while (Date.now() < deadline) { + try { + const response = await fetchImpl(mcpUrl, { + method: "HEAD", + redirect: "error", + signal: AbortSignal.timeout(5_000), + }); + await response.body?.cancel(); + if (response.status === 405) return; + } catch { + // The relay and server handoff are allowed a short convergence window. + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error("performance smoke MCP endpoint did not become ready before the timeout"); +} diff --git a/test/brev-nightly-workflow.test.ts b/test/brev-nightly-workflow.test.ts index d1900631a1..e30d260ab3 100644 --- a/test/brev-nightly-workflow.test.ts +++ b/test/brev-nightly-workflow.test.ts @@ -140,6 +140,9 @@ describe("Brev nightly workflow contract", () => { it("collects and scans Brev performance smoke evidence before deleting the VM", () => { const steps = branchValidation.jobs?.["e2e-branch-validation"]?.steps ?? []; + const prerequisite = steps.find( + (step) => step.name === "Install and verify Brev performance smoke tunnel prerequisite", + ); const collect = steps.find( (step) => step.name === "Collect Brev tool-disclosure performance smoke artifacts", ); @@ -151,6 +154,13 @@ describe("Brev nightly workflow contract", () => { ); const cleanup = steps.find((step) => step.name === "Delete Brev instance"); + expect(prerequisite?.if).toContain("tool-disclosure-performance-smoke"); + expect(prerequisite?.env).toEqual({ + CLOUDFLARED_VERSION: "2026.6.1", + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526", + }); + expect(prerequisite?.run).toContain("sha256sum -c -"); + expect(prerequisite?.run).toContain("dpkg-deb -f"); expect(collect?.if).toContain("tool-disclosure-performance-smoke"); expect(collect?.run).toContain("/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"); expect(collect?.run).toContain("tar -C"); diff --git a/test/brev-remote-vitest.test.ts b/test/brev-remote-vitest.test.ts index 62aaabddfe..8ca10acfff 100644 --- a/test/brev-remote-vitest.test.ts +++ b/test/brev-remote-vitest.test.ts @@ -9,20 +9,20 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { - BREV_CLOUDFLARED_DEB_SHA256, - BREV_CLOUDFLARED_VERSION, BREV_MESSAGING_COMPAT_TIMEOUT_MS, BREV_MESSAGING_PROVIDER_TIMEOUT_MS, BREV_REMOTE_WRAPPER_GRACE_MS, BREV_SECURITY_SUITE_TIMEOUT_MS, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT, BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE, BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS, BREV_WORKFLOW_OWNERSHIP_ENV, brevSuiteHarnessSandboxName, brevSuiteNeedsHarnessSandbox, brevWorkflowOwnsInstance, - buildBrevCloudflaredInstallCommands, + buildBrevMcpSshForwardArgs, buildBrevRemoteVitestCommand, + buildBrevSshForwardEnvironment, } from "../tools/e2e/brev-remote-vitest.mts"; const TARGET = "test/e2e/live/credential-sanitization.test.ts"; @@ -104,6 +104,7 @@ describe("Brev remote Vitest command", () => { expect(BREV_MESSAGING_PROVIDER_TIMEOUT_MS).toBe(70 * 60_000); expect(BREV_MESSAGING_COMPAT_TIMEOUT_MS).toBe(40 * 60_000); expect(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS).toBe(55 * 60_000); + expect(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT).toBeGreaterThanOrEqual(1_024); expect(BREV_REMOTE_WRAPPER_GRACE_MS).toBe(120_000); }); @@ -114,6 +115,46 @@ describe("Brev remote Vitest command", () => { expect(brevWorkflowOwnsInstance({})).toBe(false); }); + it("keeps inference and workflow credentials out of the MCP SSH forward", () => { + expect( + buildBrevSshForwardEnvironment({ + PATH: "/usr/bin", + HOME: "/home/runner", + LC_ALL: "C.UTF-8", + SSH_AUTH_SOCK: "/tmp/agent.sock", + NVIDIA_INFERENCE_API_KEY: "secret-inference-key", + GITHUB_TOKEN: "secret-workflow-token", + }), + ).toEqual({ + PATH: "/usr/bin", + HOME: "/home/runner", + LC_ALL: "C.UTF-8", + }); + expect(buildBrevMcpSshForwardArgs("e2e-performance-smoke", 45_123)).toEqual([ + "-N", + "-T", + "-o", + "BatchMode=yes", + "-o", + "ForwardAgent=no", + "-o", + "StrictHostKeyChecking=no", + "-o", + "LogLevel=ERROR", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "ConnectTimeout=15", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=4", + "-L", + "127.0.0.1:45123:127.0.0.1:43117", + "e2e-performance-smoke", + ]); + }); + it("does not seed shared harness state for suites that own their sandbox lifecycle", () => { expect(brevSuiteNeedsHarnessSandbox("all")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("full")).toBe(false); @@ -129,18 +170,6 @@ describe("Brev remote Vitest command", () => { ).toBeUndefined(); }); - it("installs the smoke tunnel prerequisite from an immutable reviewed package", () => { - const command = buildBrevCloudflaredInstallCommands().join("\n"); - - expect(command).toContain(BREV_CLOUDFLARED_VERSION); - expect(command).toContain(BREV_CLOUDFLARED_DEB_SHA256); - expect(command).toContain("sha256sum -c -"); - expect(command).toContain("dpkg-deb -f"); - expect(command).toContain("sudo dpkg -i"); - expect(command).not.toContain("apt-get install"); - expect(command).not.toContain("pkg.cloudflare.com"); - }); - it("preserves harness onboarding for single-target suites", () => { expect(brevSuiteNeedsHarnessSandbox("credential-sanitization")).toBe(true); expect(brevSuiteNeedsHarnessSandbox("telegram-injection")).toBe(true); diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index fc427d27bc..af463737a7 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -50,9 +50,25 @@ * TELEGRAM_CHAT_ID_E2E — Telegram chat ID for optional sendMessage test */ -import { execFileSync, execSync, type StdioOptions, spawnSync } from "node:child_process"; +import { + type ChildProcess, + execFileSync, + execSync, + type StdioOptions, + spawn, + spawnSync, +} from "node:child_process"; +import net from "node:net"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + type QuickTunnel, + startQuickTunnel, +} from "../../scripts/performance/tool-disclosure/quick-tunnel"; +import { + PERFORMANCE_SMOKE_MCP_PORT_ENV, + PERFORMANCE_SMOKE_MCP_URL_ENV, +} from "../../scripts/performance/tool-disclosure/smoke-mcp-transport"; import { shellQuote } from "../../src/lib/core/shell-quote"; import { BREV_MESSAGING_COMPAT_TIMEOUT_MS, @@ -60,13 +76,15 @@ import { BREV_REMOTE_WRAPPER_GRACE_MS, BREV_SECURITY_SUITE_TIMEOUT_MS, BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT, BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE, BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS, brevSuiteHarnessSandboxName, brevSuiteNeedsHarnessSandbox, brevWorkflowOwnsInstance, - buildBrevCloudflaredInstallCommands, + buildBrevMcpSshForwardArgs, buildBrevRemoteVitestCommand, + buildBrevSshForwardEnvironment, } from "../../tools/e2e/brev-remote-vitest.mts"; // Instance configuration @@ -124,9 +142,15 @@ const SETUP_SCRIPT_PATH = path.join(REPO_DIR, "scripts", "brev-launchable-ci-cpu // Sentinel file written by brev-launchable-ci-cpu.sh when setup is complete. // More reliable than grepping log files. const LAUNCHABLE_SENTINEL = "/var/run/nemoclaw-launchable-ready"; +const PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE = + "/tmp/nemoclaw-tool-disclosure-performance-mcp-placeholder.pid"; +const PERFORMANCE_SMOKE_MCP_PLACEHOLDER_LOG_FILE = + "/tmp/nemoclaw-tool-disclosure-performance-mcp-placeholder.log"; let remoteDir = ""; let instanceCreated = false; +let performanceSmokeMcpForward: ChildProcess | undefined; +let performanceSmokeMcpTunnel: QuickTunnel | undefined; const STREAM_STDIO: StdioOptions = ["inherit", "inherit", "inherit"]; const CAPTURE_STDIO: StdioOptions = ["pipe", "pipe", "pipe"]; @@ -296,6 +320,9 @@ function sshEnv( envParts.push(`export NEMOCLAW_MODEL='${shellEscape(gpuE2eModel)}'`); } if (TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE) { + if (!performanceSmokeMcpTunnel) { + throw new Error("Brev performance smoke MCP tunnel is not ready"); + } const testedSha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: REPO_DIR, encoding: "utf8", @@ -306,6 +333,8 @@ function sshEnv( envParts.push( `export E2E_ARTIFACT_DIR='${shellEscape(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR)}'`, `export GITHUB_SHA='${testedSha}'`, + `export ${PERFORMANCE_SMOKE_MCP_PORT_ENV}='${BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT}'`, + `export ${PERFORMANCE_SMOKE_MCP_URL_ENV}='${shellEscape(performanceSmokeMcpTunnel.mcpUrl)}'`, ); } // Forward optional messaging tokens for the messaging-providers test @@ -786,13 +815,127 @@ function prepareGpuDockerRuntime(elapsed: () => string): void { console.log(`[${elapsed()}] NVIDIA Docker runtime ready`); } -function prepareToolDisclosurePerformanceSmoke(elapsed: () => string): void { - console.log(`[${elapsed()}] Installing verified cloudflared prerequisite...`); - ssh(buildBrevCloudflaredInstallCommands().join(" && "), { - timeout: 180_000, - stream: true, +function performanceSmokePlaceholderSource(): string { + return [ + "from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer", + "class Handler(BaseHTTPRequestHandler):", + " def do_HEAD(self):", + " self.send_response(405)", + " self.end_headers()", + " def do_GET(self):", + " self.send_response(405)", + " self.end_headers()", + " def log_message(self, format, *args):", + " pass", + `ThreadingHTTPServer((\"127.0.0.1\", ${BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT}), Handler).serve_forever()`, + ].join("\n"); +} + +function startPerformanceSmokeMcpPlaceholder(): void { + ssh( + [ + "set -euo pipefail", + `rm -f ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE} ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_LOG_FILE}`, + `nohup python3 -c ${shellQuote(performanceSmokePlaceholderSource())} ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_LOG_FILE} 2>&1 &`, + "placeholder_pid=$!", + `printf '%s\\n' \"$placeholder_pid\" >${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE}`, + "sleep 1", + 'kill -0 "$placeholder_pid"', + ].join("\n"), + { timeout: 15_000 }, + ); +} + +function stopPerformanceSmokeMcpPlaceholder(): void { + ssh( + [ + "set +e", + `if test -s ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE}; then`, + `placeholder_pid=\"$(cat ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE})\"`, + 'kill "$placeholder_pid" 2>/dev/null', + "for attempt in 1 2 3 4 5; do", + 'if ! kill -0 "$placeholder_pid" 2>/dev/null; then break; fi', + "sleep 1", + "done", + 'kill -9 "$placeholder_pid" 2>/dev/null', + "fi", + `rm -f ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE}`, + ].join("\n"), + { timeout: 15_000 }, + ); +} + +function reserveLoopbackPort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + reject(new Error("failed to reserve a loopback port for the Brev MCP forward")); + return; + } + const port = address.port; + server.close((error) => (error ? reject(error) : resolve(port))); + }); + }); +} + +function canConnectLoopback(port: number): Promise { + return new Promise((resolve) => { + const socket = net.createConnection({ host: "127.0.0.1", port }); + const finish = (connected: boolean): void => { + socket.removeAllListeners(); + socket.destroy(); + resolve(connected); + }; + socket.setTimeout(500); + socket.once("connect", () => finish(true)); + socket.once("error", () => finish(false)); + socket.once("timeout", () => finish(false)); }); - console.log(`[${elapsed()}] Verified cloudflared prerequisite ready`); +} + +async function stopManagedChild(child: ChildProcess | undefined): Promise { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const closed = new Promise((resolve) => child.once("close", () => resolve())); + child.kill("SIGTERM"); + await Promise.race([closed, new Promise((resolve) => setTimeout(resolve, 5_000))]); + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); +} + +async function startPerformanceSmokeSshForward(localPort: number): Promise { + const child = spawn("ssh", buildBrevMcpSshForwardArgs(requireInstanceName(), localPort), { + env: buildBrevSshForwardEnvironment(process.env), + stdio: ["ignore", "ignore", "inherit"], + }); + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error("Brev MCP SSH forward exited before readiness"); + } + if (await canConnectLoopback(localPort)) return child; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + await stopManagedChild(child); + throw new Error("Brev MCP SSH forward did not become ready"); +} + +async function prepareToolDisclosurePerformanceSmoke(elapsed: () => string): Promise { + console.log(`[${elapsed()}] Preparing runner-relayed MCP tunnel...`); + startPerformanceSmokeMcpPlaceholder(); + const localPort = await reserveLoopbackPort(); + try { + performanceSmokeMcpForward = await startPerformanceSmokeSshForward(localPort); + performanceSmokeMcpTunnel = await startQuickTunnel({ port: localPort, timeoutMs: 90_000 }); + } catch (error) { + await stopManagedChild(performanceSmokeMcpForward); + performanceSmokeMcpForward = undefined; + stopPerformanceSmokeMcpPlaceholder(); + throw error; + } + console.log(`[${elapsed()}] Runner-relayed MCP tunnel ready`); } /** @@ -1123,7 +1266,7 @@ describe("Brev GPU runtime setup", () => { }); describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { - beforeAll(() => { + beforeAll(async () => { const bootstrapStart = Date.now(); const elapsed = () => `${Math.round((Date.now() - bootstrapStart) / 1000)}s`; @@ -1155,7 +1298,7 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { remoteDir = result.remoteDir; if (TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE) { - prepareToolDisclosurePerformanceSmoke(elapsed); + await prepareToolDisclosurePerformanceSmoke(elapsed); } if (result.needsOnboard) { @@ -1182,7 +1325,16 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { console.log(`[${elapsed()}] beforeAll complete — total bootstrap time: ${elapsed()}`); }, 2_700_000); // 45 min - afterAll(() => { + afterAll(async () => { + if (performanceSmokeMcpTunnel) { + await performanceSmokeMcpTunnel.close(); + performanceSmokeMcpTunnel = undefined; + } + await stopManagedChild(performanceSmokeMcpForward); + performanceSmokeMcpForward = undefined; + if (instanceCreated && TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE) { + stopPerformanceSmokeMcpPlaceholder(); + } if (!instanceCreated) return; const keepAlive = process.env.KEEP_ALIVE === "true"; const workflowOwnsInstance = brevWorkflowOwnsInstance(); @@ -1224,6 +1376,7 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { it.runIf(TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE)( "tool-disclosure performance smoke passes on Brev CPU VM", () => { + stopPerformanceSmokeMcpPlaceholder(); const output = runRemoteVitest( "e2e-live", "test/e2e/live/tool-disclosure-performance-smoke.test.ts", diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 814895f066..ddb21e3bec 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -23,6 +23,10 @@ import { SyntheticMcpServer } from "../../../scripts/performance/tool-disclosure import { startQuickTunnel } from "../../../scripts/performance/tool-disclosure/quick-tunnel"; import { createToolDisclosureRecordingProxy } from "../../../scripts/performance/tool-disclosure/recorder"; import type { ToolDisclosureMode } from "../../../scripts/performance/tool-disclosure/schedule"; +import { + resolvePerformanceSmokeMcpTransport, + waitForPerformanceSmokeMcpEndpoint, +} from "../../../scripts/performance/tool-disclosure/smoke-mcp-transport"; import { generatePrimaryTaskSet } from "../../../scripts/performance/tool-disclosure/tasks"; import { LOCAL_INFERENCE_TIMEOUT_SECS } from "../../../src/lib/onboard/env"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -120,6 +124,7 @@ test("tool disclosure hosted-inference performance smoke completes one frozen ta }); const catalogPrefix = generateCatalogPrefix(catalog, CATALOG_SIZE); const primaryTasks = generatePrimaryTaskSet(catalog); + const mcpTransport = resolvePerformanceSmokeMcpTransport(); const task = primaryTasks.tasks.find((candidate) => candidate.id === TASK_ID); expect(task, `frozen task ${TASK_ID} is missing`).toBeDefined(); const frozenTask = task as NonNullable; @@ -138,13 +143,27 @@ test("tool disclosure hosted-inference performance smoke completes one frozen ta catalog_size: CATALOG_SIZE, task_id: TASK_ID, model_id: hosted.model, + mcp_transport: mcpTransport.kind, }); const mcp = new SyntheticMcpServer(catalogPrefix, bearerToken); - const mcpAddress = await mcp.start(); + const mcpAddress = await mcp.start( + mcpTransport.kind === "external-ssh-forward" ? mcpTransport.listenPort : 0, + ); cleanup.add("stop tool-disclosure performance synthetic MCP server", () => mcp.stop()); - const tunnel = await startQuickTunnel({ port: mcpAddress.port }); - cleanup.add("stop tool-disclosure performance MCP quick tunnel", () => tunnel.close()); + const localTunnel = + mcpTransport.kind === "local-quick-tunnel" + ? await startQuickTunnel({ port: mcpAddress.port }) + : undefined; + if (localTunnel) { + cleanup.add("stop tool-disclosure performance MCP quick tunnel", () => localTunnel.close()); + } + const mcpUrl = + mcpTransport.kind === "external-ssh-forward" ? mcpTransport.mcpUrl : localTunnel?.mcpUrl; + expect(mcpUrl, "performance smoke MCP URL was not established").toBeDefined(); + const configuredMcpUrl = mcpUrl as string; + artifacts.addRedactionValues([configuredMcpUrl]); + await waitForPerformanceSmokeMcpEndpoint(configuredMcpUrl); for (const mode of MODES) { const name = sandboxName(mode); @@ -208,14 +227,14 @@ test("tool disclosure hosted-inference performance smoke completes one frozen ta "add", MCP_SERVER_NAME, "--url", - tunnel.mcpUrl, + configuredMcpUrl, "--env", MCP_TOKEN_ENV, ], { artifactName: `add-tool-disclosure-mcp-${mode}`, env: hostEnv({ [MCP_TOKEN_ENV]: bearerToken }), - redactionValues: [bearerToken], + redactionValues: [bearerToken, configuredMcpUrl], timeoutMs: 5 * 60_000, }, ); @@ -552,12 +571,18 @@ test("tool disclosure hosted-inference performance smoke completes one frozen ta api: "openai-compatible", model_id: hosted.model, }, + mcp_transport: mcpTransport.kind, results, limitations: [ "This performance smoke test verifies live wiring and task completion; it is not the complete two-campaign performance test.", "One observation per mode is insufficient for performance or quality claims.", "The fixed progressive-then-direct order makes elapsed times informational and subject to cold-cache and ordering effects.", "The performance smoke test does not collect vLLM tokenizer, token-counter, or request-recorder evidence.", + ...(mcpTransport.kind === "external-ssh-forward" + ? [ + "The MCP HTTPS endpoint is relayed through the CI runner over an authenticated SSH connection to the host-local synthetic server.", + ] + : []), ], }); diff --git a/test/performance/tool-disclosure-quick-tunnel.test.ts b/test/performance/tool-disclosure-quick-tunnel.test.ts index 0cb00660cd..45fc02459b 100644 --- a/test/performance/tool-disclosure-quick-tunnel.test.ts +++ b/test/performance/tool-disclosure-quick-tunnel.test.ts @@ -1,12 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; import { buildQuickTunnelArgs, buildQuickTunnelEnvironment, parseQuickTunnelOrigin, + startQuickTunnel, } from "../../scripts/performance/tool-disclosure/quick-tunnel"; describe("tool-disclosure quick tunnel", () => { @@ -60,4 +64,38 @@ describe("tool-disclosure quick tunnel", () => { ]); expect(() => buildQuickTunnelArgs(0)).toThrow("between 1 and 65535"); }); + + it("discovers a tunnel origin from the bounded child log and closes the child", async () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-quick-tunnel-test-")); + const binary = path.join(fixture, "fake-cloudflared"); + fs.writeFileSync( + binary, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "printf '%s\\n' 'INF Requesting new quick Tunnel https://log-backed-123.trycloudflare.com' >&2", + "trap 'exit 0' TERM INT", + "while :; do sleep 1; done", + "", + ].join("\n"), + { mode: 0o755 }, + ); + + try { + const tunnel = await startQuickTunnel({ + port: 31_337, + binary, + env: { PATH: process.env.PATH }, + fetchImpl: (async (input) => { + expect(String(input)).toBe("https://log-backed-123.trycloudflare.com/mcp"); + return new Response(null, { status: 405 }); + }) as typeof fetch, + timeoutMs: 5_000, + }); + expect(tunnel.mcpUrl).toBe("https://log-backed-123.trycloudflare.com/mcp"); + await tunnel.close(); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); }); diff --git a/test/performance/tool-disclosure-smoke-mcp-transport.test.ts b/test/performance/tool-disclosure-smoke-mcp-transport.test.ts new file mode 100644 index 0000000000..dd4f5723a8 --- /dev/null +++ b/test/performance/tool-disclosure-smoke-mcp-transport.test.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + PERFORMANCE_SMOKE_MCP_PORT_ENV, + PERFORMANCE_SMOKE_MCP_URL_ENV, + resolvePerformanceSmokeMcpTransport, + waitForPerformanceSmokeMcpEndpoint, +} from "../../scripts/performance/tool-disclosure/smoke-mcp-transport"; + +function externalEnv(url = "https://forwarded-smoke-123.trycloudflare.com/mcp") { + return { + [PERFORMANCE_SMOKE_MCP_URL_ENV]: url, + [PERFORMANCE_SMOKE_MCP_PORT_ENV]: "43117", + }; +} + +describe("tool-disclosure performance smoke MCP transport", () => { + it("uses the local quick tunnel when no external transport is configured", () => { + expect(resolvePerformanceSmokeMcpTransport({})).toEqual({ kind: "local-quick-tunnel" }); + }); + + it("accepts only a paired exact external tunnel URL and loopback listener port", () => { + expect(resolvePerformanceSmokeMcpTransport(externalEnv())).toEqual({ + kind: "external-ssh-forward", + mcpUrl: "https://forwarded-smoke-123.trycloudflare.com/mcp", + listenPort: 43117, + }); + expect(() => + resolvePerformanceSmokeMcpTransport({ + [PERFORMANCE_SMOKE_MCP_URL_ENV]: externalEnv()[PERFORMANCE_SMOKE_MCP_URL_ENV], + }), + ).toThrow("configured together"); + }); + + it.each([ + "http://forwarded-smoke-123.trycloudflare.com/mcp", + "https://forwarded-smoke-123.trycloudflare.com/other", + "https://forwarded-smoke-123.trycloudflare.com/mcp?token=x", + "https://user:pass@forwarded-smoke-123.trycloudflare.com/mcp", + "https://forwarded-smoke-123.trycloudflare.com.attacker.invalid/mcp", + ])("rejects unsafe external URL %s", (url) => { + expect(() => resolvePerformanceSmokeMcpTransport(externalEnv(url))).toThrow( + "exact trycloudflare /mcp URL", + ); + }); + + it.each(["0", "1023", "65536", "12x"])("rejects unsafe external listener port %s", (port) => { + expect(() => + resolvePerformanceSmokeMcpTransport({ + ...externalEnv(), + [PERFORMANCE_SMOKE_MCP_PORT_ENV]: port, + }), + ).toThrow(/port/); + }); + + it("waits through relay handoff responses until the MCP endpoint is ready", async () => { + const statuses = [502, 405]; + await expect( + waitForPerformanceSmokeMcpEndpoint("https://forwarded-smoke-123.trycloudflare.com/mcp", { + fetchImpl: (() => + Promise.resolve(new Response(null, { status: statuses.shift() ?? 405 }))) as typeof fetch, + timeoutMs: 1_000, + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/tools/e2e/brev-remote-vitest.mts b/tools/e2e/brev-remote-vitest.mts index 82994695a3..5ae828690d 100644 --- a/tools/e2e/brev-remote-vitest.mts +++ b/tools/e2e/brev-remote-vitest.mts @@ -14,9 +14,7 @@ export const BREV_WORKFLOW_OWNERSHIP_ENV = "NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE = "tool-disclosure-performance-smoke"; export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR = "/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"; -export const BREV_CLOUDFLARED_VERSION = "2026.6.1"; -export const BREV_CLOUDFLARED_DEB_SHA256 = - "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"; +export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT = 43_117; const BREV_SUITES_WITHOUT_HARNESS_SANDBOX = new Set([ "all", @@ -39,6 +37,47 @@ export function brevWorkflowOwnsInstance(env: NodeJS.ProcessEnv = process.env): return env[BREV_WORKFLOW_OWNERSHIP_ENV] === "1"; } +export function buildBrevSshForwardEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const selected: NodeJS.ProcessEnv = {}; + for (const name of ["PATH", "HOME", "USER", "LOGNAME", "LANG"]) { + if (env[name] !== undefined) selected[name] = env[name]; + } + for (const [name, value] of Object.entries(env)) { + if (name.startsWith("LC_") && value !== undefined) selected[name] = value; + } + return selected; +} + +export function buildBrevMcpSshForwardArgs(instanceName: string, localPort: number): string[] { + if (!instanceName) throw new Error("Brev MCP SSH forward requires an instance name"); + if (!Number.isInteger(localPort) || localPort < 1_024 || localPort > 65_535) { + throw new Error("Brev MCP SSH forward local port must be between 1024 and 65535"); + } + return [ + "-N", + "-T", + "-o", + "BatchMode=yes", + "-o", + "ForwardAgent=no", + "-o", + "StrictHostKeyChecking=no", + "-o", + "LogLevel=ERROR", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "ConnectTimeout=15", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=4", + "-L", + `127.0.0.1:${localPort}:127.0.0.1:${BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT}`, + instanceName, + ]; +} + export function buildBrevRemoteVitestCommand(project: BrevVitestProject, target: string): string { const vitestCommand = [ "./node_modules/.bin/vitest", @@ -62,17 +101,3 @@ export function buildBrevRemoteVitestCommand(project: BrevVitestProject, target: `NEMOCLAW_RUN_LIVE_E2E=1 ${vitestCommand}`, ].join(" && "); } - -export function buildBrevCloudflaredInstallCommands(): string[] { - return [ - "set -euo pipefail", - `cloudflared_deb=/tmp/cloudflared-${BREV_CLOUDFLARED_VERSION}-linux-amd64.deb`, - `curl -fL https://github.com/cloudflare/cloudflared/releases/download/${BREV_CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb -o \"$cloudflared_deb\"`, - `printf '%s %s\\n' ${BREV_CLOUDFLARED_DEB_SHA256} \"$cloudflared_deb\" | sha256sum -c -`, - `test \"$(dpkg-deb -f \"$cloudflared_deb\" Package)\" = cloudflared`, - `test \"$(dpkg-deb -f \"$cloudflared_deb\" Version)\" = ${BREV_CLOUDFLARED_VERSION}`, - `test \"$(dpkg-deb -f \"$cloudflared_deb\" Architecture)\" = amd64`, - `sudo dpkg -i \"$cloudflared_deb\"`, - `cloudflared --version | grep -F 'cloudflared version ${BREV_CLOUDFLARED_VERSION}'`, - ]; -} From c064f4af06843f56f811eb9cc617b56c444e5fa6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 13:59:19 -0700 Subject: [PATCH 27/31] ci: diagnose Brev tunnel readiness Signed-off-by: Aaron Erickson --- .github/workflows/e2e-branch-validation.yaml | 32 ++++++++++--- .../tool-disclosure/quick-tunnel.ts | 47 +++++++++++++++---- test/brev-nightly-workflow.test.ts | 4 +- test/e2e/brev-e2e.test.ts | 6 ++- .../tool-disclosure-quick-tunnel.test.ts | 1 + 5 files changed, 73 insertions(+), 17 deletions(-) diff --git a/.github/workflows/e2e-branch-validation.yaml b/.github/workflows/e2e-branch-validation.yaml index d4088f65e0..eedeccc3a8 100644 --- a/.github/workflows/e2e-branch-validation.yaml +++ b/.github/workflows/e2e-branch-validation.yaml @@ -291,7 +291,8 @@ jobs: sudo dpkg -i "${cloudflared_deb}" cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" - - name: Run ephemeral Brev E2E + - id: brev-test + name: Run ephemeral Brev E2E env: NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1" NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE: "1" @@ -314,17 +315,34 @@ jobs: if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke' env: INSTANCE: ${{ env.BREV_E2E_INSTANCE_NAME }} + TEST_OUTCOME: ${{ steps.brev-test.outcome }} run: | set -euo pipefail artifact_dir="brev-tool-disclosure-performance-smoke-artifacts" remote_dir="/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts" + archive="${RUNNER_TEMP}/brev-tool-disclosure-performance-smoke-artifacts.tar.gz" mkdir -p "$artifact_dir" brev refresh >/dev/null - ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR \ - -o ConnectTimeout=10 "$INSTANCE" \ - "test -d '$remote_dir' && tar -C '$remote_dir' -czf - ." \ - | tar -C "$artifact_dir" -xzf - - test -f "$artifact_dir/tool-disclosure-hosted-inference-performance-smoke-completes-one-frozen-task-in-both-modes/tool-disclosure-performance-smoke.json" + if ! ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR \ + -o ConnectTimeout=10 "$INSTANCE" \ + "test -d '$remote_dir' && tar -C '$remote_dir' -czf - ." \ + >"$archive"; then + if [ "$TEST_OUTCOME" = "success" ]; then + echo "::error::Successful Brev performance smoke did not produce an artifact archive" + exit 1 + fi + echo "::warning::Failed Brev performance smoke produced no artifact archive" + exit 0 + fi + tar -C "$artifact_dir" -xzf "$archive" + result="$artifact_dir/tool-disclosure-hosted-inference-performance-smoke-completes-one-frozen-task-in-both-modes/tool-disclosure-performance-smoke.json" + if [ ! -f "$result" ]; then + if [ "$TEST_OUTCOME" = "success" ]; then + echo "::error::Successful Brev performance smoke artifact is missing its result JSON" + exit 1 + fi + echo "::warning::Failed Brev performance smoke has only partial artifacts" + fi - name: Scan Brev tool-disclosure performance smoke artifacts if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke' @@ -346,7 +364,7 @@ jobs: with: name: e2e-brev-tool-disclosure-performance-smoke-${{ github.run_attempt }} path: brev-tool-disclosure-performance-smoke-artifacts/ - if-no-files-found: error + if-no-files-found: ignore include-hidden-files: false retention-days: 14 diff --git a/scripts/performance/tool-disclosure/quick-tunnel.ts b/scripts/performance/tool-disclosure/quick-tunnel.ts index 5f07bf373e..89e57cbb17 100644 --- a/scripts/performance/tool-disclosure/quick-tunnel.ts +++ b/scripts/performance/tool-disclosure/quick-tunnel.ts @@ -31,11 +31,16 @@ export interface QuickTunnel { close(): Promise; } +export type QuickTunnelProtocol = "auto" | "http2" | "quic"; + export function parseQuickTunnelOrigin(text: string): string | null { return [...text.matchAll(ORIGIN_PATTERN)].at(-1)?.[0] ?? null; } -export function buildQuickTunnelArgs(port: number): string[] { +export function buildQuickTunnelArgs( + port: number, + protocol: QuickTunnelProtocol = "http2", +): string[] { if (!Number.isInteger(port) || port < 1 || port > 65_535) { throw new Error("quick-tunnel origin port must be between 1 and 65535"); } @@ -44,7 +49,7 @@ export function buildQuickTunnelArgs(port: number): string[] { "--config=", "--no-autoupdate", "--protocol", - "http2", + protocol, "--url", `http://127.0.0.1:${port}`, "--loglevel", @@ -106,7 +111,7 @@ function readLogTail(file: string, maxBytes = 8_192): string { } } -async function probe(origin: string, fetchImpl: typeof fetch): Promise { +async function probe(origin: string, fetchImpl: typeof fetch): Promise { try { const response = await fetchImpl(`${origin}/mcp`, { method: "HEAD", @@ -114,20 +119,29 @@ async function probe(origin: string, fetchImpl: typeof fetch): Promise signal: AbortSignal.timeout(5_000), }); await response.body?.cancel(); - return response.status === 405; + return response.status; } catch { - return false; + return null; } } +function sanitizedLogDiagnostic(file: string): string { + return readLogTail(file, 2_048) + .replace(ORIGIN_PATTERN, "https://redacted.trycloudflare.com") + .replace(/\s+/gu, " ") + .trim() + .slice(-1_500); +} + export async function startQuickTunnel(options: { port: number; binary?: string; timeoutMs?: number; fetchImpl?: typeof fetch; env?: NodeJS.ProcessEnv; + protocol?: QuickTunnelProtocol; }): Promise { - const args = buildQuickTunnelArgs(options.port); + const args = buildQuickTunnelArgs(options.port, options.protocol); const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-")); const logFile = path.join(isolatedHome, "cloudflared.log"); const removeIsolatedHome = (): void => fs.rmSync(isolatedHome, { recursive: true, force: true }); @@ -149,6 +163,7 @@ export async function startQuickTunnel(options: { const timeoutMs = options.timeoutMs ?? 45_000; const fetchImpl = options.fetchImpl ?? fetch; let origin: string | null = null; + let lastProbeStatus: number | null | undefined; let spawnError: Error | undefined; child.once("error", (error) => { spawnError = error; @@ -159,7 +174,8 @@ export async function startQuickTunnel(options: { origin = parseQuickTunnelOrigin(readLogTail(logFile)) ?? origin; if (spawnError) break; if (child.exitCode !== null || child.signalCode !== null) break; - if (origin && (await probe(origin, fetchImpl))) { + if (origin) lastProbeStatus = await probe(origin, fetchImpl); + if (origin && lastProbeStatus === 405) { const published = origin; return { origin: published, @@ -175,11 +191,26 @@ export async function startQuickTunnel(options: { } await new Promise((resolve) => setTimeout(resolve, 250)); } + const diagnostic = sanitizedLogDiagnostic(logFile); + const childState = + child.exitCode !== null + ? `exit-${child.exitCode}` + : child.signalCode !== null + ? `signal-${child.signalCode}` + : "running"; try { await stopChild(child); } finally { removeIsolatedHome(); } if (spawnError) throw new Error(`cloudflared quick tunnel failed to start (${spawnError.name})`); - throw new Error("cloudflared quick tunnel did not become ready before the timeout"); + throw new Error( + [ + "cloudflared quick tunnel did not become ready before the timeout", + `origin_published=${origin !== null}`, + `last_probe_status=${lastProbeStatus === undefined ? "not-attempted" : (lastProbeStatus ?? "fetch-error")}`, + `child_state=${childState}`, + diagnostic ? `log=${diagnostic}` : "log=empty", + ].join("; "), + ); } diff --git a/test/brev-nightly-workflow.test.ts b/test/brev-nightly-workflow.test.ts index e30d260ab3..50a25ea2ea 100644 --- a/test/brev-nightly-workflow.test.ts +++ b/test/brev-nightly-workflow.test.ts @@ -162,14 +162,16 @@ describe("Brev nightly workflow contract", () => { expect(prerequisite?.run).toContain("sha256sum -c -"); expect(prerequisite?.run).toContain("dpkg-deb -f"); expect(collect?.if).toContain("tool-disclosure-performance-smoke"); + expect(collect?.env?.TEST_OUTCOME).toBe("${{ steps.brev-test.outcome }}"); expect(collect?.run).toContain("/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"); expect(collect?.run).toContain("tar -C"); + expect(collect?.run).toContain('if [ "$TEST_OUTCOME" = "success" ]'); expect(scan?.env?.NVIDIA_INFERENCE_API_KEY).toBe("${{ secrets.NVIDIA_INFERENCE_API_KEY }}"); expect(scan?.run).toContain("grep -R -I -l -F"); expect(upload?.uses).toBe("actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"); expect(upload?.with).toMatchObject({ name: "e2e-brev-tool-disclosure-performance-smoke-${{ github.run_attempt }}", - "if-no-files-found": "error", + "if-no-files-found": "ignore", "retention-days": 14, }); expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index af463737a7..3bcbb973cd 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -928,7 +928,11 @@ async function prepareToolDisclosurePerformanceSmoke(elapsed: () => string): Pro const localPort = await reserveLoopbackPort(); try { performanceSmokeMcpForward = await startPerformanceSmokeSshForward(localPort); - performanceSmokeMcpTunnel = await startQuickTunnel({ port: localPort, timeoutMs: 90_000 }); + performanceSmokeMcpTunnel = await startQuickTunnel({ + port: localPort, + protocol: "auto", + timeoutMs: 90_000, + }); } catch (error) { await stopManagedChild(performanceSmokeMcpForward); performanceSmokeMcpForward = undefined; diff --git a/test/performance/tool-disclosure-quick-tunnel.test.ts b/test/performance/tool-disclosure-quick-tunnel.test.ts index 45fc02459b..c06dbaf1fa 100644 --- a/test/performance/tool-disclosure-quick-tunnel.test.ts +++ b/test/performance/tool-disclosure-quick-tunnel.test.ts @@ -63,6 +63,7 @@ describe("tool-disclosure quick tunnel", () => { "info", ]); expect(() => buildQuickTunnelArgs(0)).toThrow("between 1 and 65535"); + expect(buildQuickTunnelArgs(31337, "auto")).toContain("auto"); }); it("discovers a tunnel origin from the bounded child log and closes the child", async () => { From 0c56519509c65d916e85318dd13e5a291d8d25a3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 14:12:00 -0700 Subject: [PATCH 28/31] ci: size Brev performance smoke host Signed-off-by: Aaron Erickson --- .github/workflows/e2e-branch-validation.yaml | 11 ++++++++++- test/brev-nightly-workflow.test.ts | 7 +++++++ test/e2e/brev-e2e.test.ts | 10 ++++++++++ .../live/tool-disclosure-performance-smoke.test.ts | 9 +++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-branch-validation.yaml b/.github/workflows/e2e-branch-validation.yaml index eedeccc3a8..5170363dfb 100644 --- a/.github/workflows/e2e-branch-validation.yaml +++ b/.github/workflows/e2e-branch-validation.yaml @@ -55,7 +55,8 @@ name: E2E / Branch Validation # its own sandbox lifecycle (NOT the independent full journey). # # Required secrets: BREV_API_KEY + BREV_ORG_ID (or legacy BREV_API_TOKEN), NVIDIA_INFERENCE_API_KEY -# Instance cost: Brev CPU credits (~$0.10/run for 4x16 instance) +# Instance cost: Brev CPU credits; most suites use 4x16, while the performance +# smoke requests 8 vCPU, 32 GB RAM, and 100 GB disk. on: workflow_dispatch: @@ -302,6 +303,9 @@ jobs: INSTANCE_NAME: ${{ env.BREV_E2E_INSTANCE_NAME }} TEST_SUITE: ${{ inputs.test_suite }} USE_LAUNCHABLE: ${{ inputs.use_launchable && '1' || '0' }} + BREV_MIN_VCPU: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '8' || '' }} + BREV_MIN_RAM: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '32' || '' }} + BREV_MIN_DISK: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '100' || '' }} BREV_PROVIDER: ${{ inputs.brev_provider || vars.BREV_PROVIDER || '' }} BREV_GPU_TYPE: ${{ inputs.brev_gpu_type || vars.BREV_GPU_TYPE || '' }} BREV_GPU_NAME: ${{ inputs.brev_gpu_name || vars.BREV_GPU_NAME || '' }} @@ -388,6 +392,11 @@ jobs: timeout 15s openshell sandbox list > /tmp/nc-debug/sandbox-list.txt 2>&1 timeout 15s openshell gateway status > /tmp/nc-debug/gateway-status.txt 2>&1 timeout 15s docker ps -a > /tmp/nc-debug/docker-ps.txt 2>&1 + free -h > /tmp/nc-debug/memory.txt 2>&1 + df -h > /tmp/nc-debug/disk.txt 2>&1 + timeout 15s sudo systemctl status docker --no-pager > /tmp/nc-debug/docker-service.txt 2>&1 + timeout 15s sudo journalctl -u docker --since "15 minutes ago" --no-pager > /tmp/nc-debug/docker-journal.txt 2>&1 + timeout 15s sudo dmesg | tail -200 > /tmp/nc-debug/kernel-tail.txt 2>&1 timeout 30s tar -C /tmp -czf /tmp/nc-debug.tar.gz nc-debug ' || true scp -o StrictHostKeyChecking=no -o LogLevel=ERROR \ diff --git a/test/brev-nightly-workflow.test.ts b/test/brev-nightly-workflow.test.ts index 50a25ea2ea..2f8aef763f 100644 --- a/test/brev-nightly-workflow.test.ts +++ b/test/brev-nightly-workflow.test.ts @@ -153,6 +153,7 @@ describe("Brev nightly workflow contract", () => { (step) => step.name === "Upload Brev tool-disclosure performance smoke artifacts", ); const cleanup = steps.find((step) => step.name === "Delete Brev instance"); + const run = steps.find((step) => step.name === "Run ephemeral Brev E2E"); expect(prerequisite?.if).toContain("tool-disclosure-performance-smoke"); expect(prerequisite?.env).toEqual({ @@ -161,6 +162,12 @@ describe("Brev nightly workflow contract", () => { }); expect(prerequisite?.run).toContain("sha256sum -c -"); expect(prerequisite?.run).toContain("dpkg-deb -f"); + expect(run?.env).toMatchObject({ + BREV_MIN_VCPU: "${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '8' || '' }}", + BREV_MIN_RAM: "${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '32' || '' }}", + BREV_MIN_DISK: + "${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '100' || '' }}", + }); expect(collect?.if).toContain("tool-disclosure-performance-smoke"); expect(collect?.env?.TEST_OUTCOME).toBe("${{ steps.brev-test.outcome }}"); expect(collect?.run).toContain("/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"); diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index 3bcbb973cd..4fbaebd698 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -504,6 +504,16 @@ function printRemoteFailureDiagnostics(): void { `PATH=$HOME/.local/bin:$PATH openshell sandbox list 2>&1 || true`, `echo "--- docker ps ---"`, `sg docker -c 'docker ps -a --filter label=openshell.ai/managed-by=openshell' 2>&1 || true`, + `echo "--- memory ---"`, + `free -h 2>&1 || true`, + `echo "--- disk ---"`, + `df -h 2>&1 || true`, + `echo "--- docker service ---"`, + `sudo systemctl status docker --no-pager 2>&1 || true`, + `echo "--- recent docker journal ---"`, + `sudo journalctl -u docker --since "15 minutes ago" --no-pager 2>&1 | tail -200 || true`, + `echo "--- recent kernel messages ---"`, + `sudo dmesg 2>&1 | tail -100 || true`, `echo "--- openshell gateway log ---"`, `tail -200 "$HOME/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.log" 2>&1 || true`, `latest="$(find "$HOME/.nemoclaw/onboard-failures" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -1)"`, diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index ddb21e3bec..4abd957282 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -3,6 +3,7 @@ import { execFileSync } from "node:child_process"; import { randomBytes } from "node:crypto"; +import os from "node:os"; import { DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, generateCatalogPrefix, @@ -572,6 +573,14 @@ test("tool disclosure hosted-inference performance smoke completes one frozen ta model_id: hosted.model, }, mcp_transport: mcpTransport.kind, + host: { + platform: process.platform, + release: os.release(), + architecture: process.arch, + cpu_model: os.cpus()[0]?.model ?? "unreported", + logical_cpu_count: os.cpus().length, + total_memory_bytes: os.totalmem(), + }, results, limitations: [ "This performance smoke test verifies live wiring and task completion; it is not the complete two-campaign performance test.", From cfcf7a97def0ff3e54077a36c365c384a67b97b1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 14:32:34 -0700 Subject: [PATCH 29/31] fix: support anonymous MCP snapshots on tmpfs Signed-off-by: Aaron Erickson --- .../managed-dcode-runtime.py | 26 +++++- ...pagents-code-managed-mcp-hardening.test.ts | 90 +++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/agents/langchain-deepagents-code/managed-dcode-runtime.py b/agents/langchain-deepagents-code/managed-dcode-runtime.py index cf0abf2b5c..f1626b6ae3 100644 --- a/agents/langchain-deepagents-code/managed-dcode-runtime.py +++ b/agents/langchain-deepagents-code/managed-dcode-runtime.py @@ -52,7 +52,13 @@ _MCP_CHILD_BINDING_ENV = "NEMOCLAW_DCODE_MCP_BINDING" _MCP_SEALED_KIND = "sealed-memfd" _MCP_ANONYMOUS_KIND = "anonymous-otmpfile" -_MCP_ANONYMOUS_DIRECTORY = Path("/tmp") +_MCP_ANONYMOUS_DIRECTORIES = (Path("/tmp"), Path("/dev/shm")) +_MCP_ANONYMOUS_DIRECTORY_FALLBACK_ERRNOS = { + errno.EINVAL, + errno.ENOENT, + errno.ENOSYS, + errno.EOPNOTSUPP, +} _MCP_FALLBACK_ERRNOS = { errno.EACCES, errno.EINVAL, @@ -699,13 +705,29 @@ def _sealed_managed_mcp_snapshot(payload: bytes) -> int: raise +def _open_anonymous_managed_mcp_writer(flags: int) -> int: + for index, directory in enumerate(_MCP_ANONYMOUS_DIRECTORIES): + try: + return os.open(directory, flags, 0o600) + except OSError as exc: + is_last = index == len(_MCP_ANONYMOUS_DIRECTORIES) - 1 + if ( + is_last + or exc.errno not in _MCP_ANONYMOUS_DIRECTORY_FALLBACK_ERRNOS + ): + raise + raise RuntimeError( + "managed MCP config requires anonymous O_TMPFILE support" + ) + + def _anonymous_managed_mcp_snapshot(payload: bytes) -> int: writer: int | None = None reader: int | None = None complete = False try: flags = os.O_TMPFILE | os.O_EXCL | os.O_RDWR | os.O_CLOEXEC - writer = os.open(_MCP_ANONYMOUS_DIRECTORY, flags, 0o600) + writer = _open_anonymous_managed_mcp_writer(flags) remaining = memoryview(payload) while remaining: written = os.write(writer, remaining) diff --git a/test/langchain-deepagents-code-managed-mcp-hardening.test.ts b/test/langchain-deepagents-code-managed-mcp-hardening.test.ts index 03a5202112..c6313a6916 100644 --- a/test/langchain-deepagents-code-managed-mcp-hardening.test.ts +++ b/test/langchain-deepagents-code-managed-mcp-hardening.test.ts @@ -136,10 +136,21 @@ assert payload is not None def blocked_memfd(*_args, **_kwargs): raise PermissionError(errno.EPERM, "blocked by seccomp") +real_open = managed.os.open +anonymous_attempts = [] + +def force_tmpfile_to_shm(path, flags, *args, **kwargs): + if flags & os.O_TMPFILE: + anonymous_attempts.append(str(path)) + if str(path) == "/tmp": + raise OSError(errno.EOPNOTSUPP, "O_TMPFILE unavailable") + return real_open(path, flags, *args, **kwargs) + with tempfile.TemporaryDirectory() as tempdir: managed._MCP_CONFIG_FILE = Path(tempdir) / ".nemoclaw-mcp.json" managed._read_managed_mcp_config = lambda: raw managed.os.memfd_create = blocked_memfd + managed.os.open = force_tmpfile_to_shm snapshot_path = managed.managed_mcp_config_path() assert snapshot_path is not None descriptor = int(snapshot_path.removeprefix("/proc/self/fd/")) @@ -220,6 +231,8 @@ print(child.managed_mcp_config_bytes(sys.argv[2]).decode(), end="") else: raise AssertionError("same-size anonymous descriptor overwrite was accepted") os.close(descriptor) +managed.os.open = real_open +assert anonymous_attempts == ["/tmp", "/dev/shm", "/tmp", "/dev/shm"] print("anonymous-fallback-ok") `); @@ -332,4 +345,81 @@ print("fallback-fail-closed-ok") expect(result.status, result.stderr).toBe(0); expect(result.stdout.trim()).toBe("fallback-fail-closed-ok"); }); + + it("tries the bounded anonymous tmpfs fallback only for unsupported filesystems", () => { + const result = runManagedHelper(String.raw` +import errno +import importlib.util +import os +import sys +from pathlib import Path + +spec = importlib.util.spec_from_file_location("_nemoclaw_managed", sys.argv[1]) +managed = importlib.util.module_from_spec(spec) +spec.loader.exec_module(managed) + +managed._MCP_ANONYMOUS_DIRECTORIES = (Path("/tmp"), Path("/dev/shm")) +real_open = managed.os.open + +for fallback_errno in managed._MCP_ANONYMOUS_DIRECTORY_FALLBACK_ERRNOS: + attempts = [] + + def filesystem_fallback(path, flags, *args, **kwargs): + attempts.append(str(path)) + if str(path) == "/tmp": + raise OSError(fallback_errno, "anonymous file unavailable") + return 123 + + managed.os.open = filesystem_fallback + try: + writer = managed._open_anonymous_managed_mcp_writer(12345) + assert writer == 123 + assert attempts == ["/tmp", "/dev/shm"] + finally: + managed.os.open = real_open + +for terminal_errno in (errno.EACCES, errno.EPERM, errno.ENOSPC, errno.EMFILE): + attempts = [] + + def terminal_error(path, flags, *args, **kwargs): + attempts.append(str(path)) + raise OSError(terminal_errno, "terminal anonymous-file error") + + managed.os.open = terminal_error + try: + try: + managed._open_anonymous_managed_mcp_writer(12345) + except OSError as exc: + assert exc.errno == terminal_errno + else: + raise AssertionError("terminal error triggered a directory fallback") + assert attempts == ["/tmp"] + finally: + managed.os.open = real_open + +attempts = [] + +def second_directory_error(path, flags, *args, **kwargs): + attempts.append(str(path)) + if str(path) == "/tmp": + raise OSError(errno.EOPNOTSUPP, "O_TMPFILE unavailable") + raise OSError(errno.ENOSPC, "tmpfs full") + +managed.os.open = second_directory_error +try: + try: + managed._open_anonymous_managed_mcp_writer(12345) + except OSError as exc: + assert exc.errno == errno.ENOSPC + else: + raise AssertionError("final directory error was masked") + assert attempts == ["/tmp", "/dev/shm"] +finally: + managed.os.open = real_open +print("anonymous-directory-fallback-ok") +`); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim()).toBe("anonymous-directory-fallback-ok"); + }); }); From 2b90fda9a690bfc89f1eed6cb16fae4e7313cdca Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 14:51:18 -0700 Subject: [PATCH 30/31] fix: permit sandbox-local MCP snapshots Signed-off-by: Aaron Erickson --- .../policy-additions.yaml | 4 +++ .../tool-disclosure-performance-smoke.test.ts | 26 +++++++++++++++++++ test/langchain-deepagents-code-image.test.ts | 3 +++ 3 files changed, 33 insertions(+) diff --git a/agents/langchain-deepagents-code/policy-additions.yaml b/agents/langchain-deepagents-code/policy-additions.yaml index 129f4501e1..d0a12c182e 100644 --- a/agents/langchain-deepagents-code/policy-additions.yaml +++ b/agents/langchain-deepagents-code/policy-additions.yaml @@ -24,6 +24,10 @@ filesystem_policy: - /sandbox - /sandbox/.deepagents - /tmp + # OpenShell blocks memfd_create; managed MCP config uses an unlinked, + # read-only O_TMPFILE snapshot on this sandbox-local tmpfs when /tmp's + # backing filesystem does not support anonymous files. + - /dev/shm - /dev/null landlock: diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index 4abd957282..d3507d8ea6 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -53,6 +53,17 @@ const ROUTED_PROXY_REQUEST_TIMEOUT_MS = 720_000; const ROUTED_GATEWAY_REQUEST_TIMEOUT_MS = 840_000; const ROUTED_AGENT_INVOCATION_TIMEOUT_MS = 900_000; const RESTORED_GATEWAY_REQUEST_TIMEOUT_MS = LOCAL_INFERENCE_TIMEOUT_SECS * 1_000; +const MANAGED_MCP_SNAPSHOT_PROBE = `import os +from deepagents_code import _nemoclaw_managed as managed + +payload = b"managed-mcp-snapshot-probe\\n" +descriptor, binding = managed._managed_mcp_snapshot(payload) +try: + assert binding["kind"] in {managed._MCP_SEALED_KIND, managed._MCP_ANONYMOUS_KIND} + assert managed._read_bound_managed_mcp_descriptor(descriptor, binding) == payload +finally: + os.close(descriptor) +print("managed-mcp-snapshot-ok:" + binding["kind"])`; function sandboxName(mode: ToolDisclosureMode): string { return `e2e-tool-disclosure-performance-${mode}`; @@ -240,6 +251,21 @@ test("tool disclosure hosted-inference performance smoke completes one frozen ta }, ); expect(add.exitCode, resultText(add)).toBe(0); + + const snapshotProbe = await sandbox.exec( + sandboxName(mode), + ["nemoclaw-start", "/opt/venv/bin/python3", "-I", "-c", MANAGED_MCP_SNAPSHOT_PROBE], + { + artifactName: `managed-mcp-snapshot-${mode}`, + env: sandboxAccessEnv(), + timeoutMs: 30_000, + }, + ); + expect(snapshotProbe.exitCode, resultText(snapshotProbe)).toBe(0); + expect([ + "managed-mcp-snapshot-ok:sealed-memfd", + "managed-mcp-snapshot-ok:anonymous-otmpfile", + ]).toContain(snapshotProbe.stdout.trim()); } const results = []; diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index 1a157bf727..e903fa5387 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -422,12 +422,15 @@ describe("LangChain Deep Agents Code image contracts", () => { it("keeps optional service egress out of the default policy and requires Landlock", () => { const policy = readAgentFile("policy-additions.yaml"); + const managedRuntime = readAgentFile("managed-dcode-runtime.py"); expect(policy).not.toContain("api.tavily.com"); expect(policy).not.toContain("api.smith.langchain.com"); expect(policy).toContain(" - /usr\n"); expect(policy).toContain(" - /opt/venv\n"); expect(policy).toContain(" - /etc\n"); + expect(policy).toContain(" - /dev/shm\n"); + expect(managedRuntime).toContain('(Path("/tmp"), Path("/dev/shm"))'); expect(policy).toContain("compatibility: strict"); expect(policy).not.toContain("compatibility: best_effort"); expect(policy).toContain("fail closed when Landlock cannot be applied"); From e9711200b1495c8a01429c810116f7f2a2719122 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 7 Jul 2026 15:02:39 -0700 Subject: [PATCH 31/31] test: keep managed snapshot probe argument safe Signed-off-by: Aaron Erickson --- .../tool-disclosure-performance-smoke.test.ts | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts index d3507d8ea6..dcc434b4a7 100644 --- a/test/e2e/live/tool-disclosure-performance-smoke.test.ts +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -53,17 +53,16 @@ const ROUTED_PROXY_REQUEST_TIMEOUT_MS = 720_000; const ROUTED_GATEWAY_REQUEST_TIMEOUT_MS = 840_000; const ROUTED_AGENT_INVOCATION_TIMEOUT_MS = 900_000; const RESTORED_GATEWAY_REQUEST_TIMEOUT_MS = LOCAL_INFERENCE_TIMEOUT_SECS * 1_000; -const MANAGED_MCP_SNAPSHOT_PROBE = `import os -from deepagents_code import _nemoclaw_managed as managed - -payload = b"managed-mcp-snapshot-probe\\n" -descriptor, binding = managed._managed_mcp_snapshot(payload) -try: - assert binding["kind"] in {managed._MCP_SEALED_KIND, managed._MCP_ANONYMOUS_KIND} - assert managed._read_bound_managed_mcp_descriptor(descriptor, binding) == payload -finally: - os.close(descriptor) -print("managed-mcp-snapshot-ok:" + binding["kind"])`; +const MANAGED_MCP_SNAPSHOT_PROBE = [ + "import os", + "from deepagents_code import _nemoclaw_managed as managed", + 'payload = b"managed-mcp-snapshot-probe\\n"', + "descriptor, binding = managed._managed_mcp_snapshot(payload)", + 'assert binding["kind"] in {managed._MCP_SEALED_KIND, managed._MCP_ANONYMOUS_KIND}', + "assert managed._read_bound_managed_mcp_descriptor(descriptor, binding) == payload", + "os.close(descriptor)", + 'print("managed-mcp-snapshot-ok:" + binding["kind"])', +].join("; "); function sandboxName(mode: ToolDisclosureMode): string { return `e2e-tool-disclosure-performance-${mode}`;