diff --git a/.github/workflows/e2e-branch-validation.yaml b/.github/workflows/e2e-branch-validation.yaml index d80d1ee877..5170363dfb 100644 --- a/.github/workflows/e2e-branch-validation.yaml +++ b/.github/workflows/e2e-branch-validation.yaml @@ -46,13 +46,17 @@ name: E2E / Branch Validation # OpenAI-compatible endpoint. Creates its own sandbox # (e2e-msg-compat) on a separate fresh instance. # dashboard-remote-bind — Verifies opt-in remote dashboard forwards bind 0.0.0.0. +# tool-disclosure-performance-smoke — Runs the claim-ineligible hosted-inference +# performance smoke on an ephemeral Brev CPU VM and +# retains its sanitized evidence. (~60 min) # gpu — Provisions a Brev GPU VM and runs the Ollama GPU E2E # sandbox proof suite from source. (~45 min) # all — Runs credential-sanitization + telegram-injection, each with # its own sandbox lifecycle (NOT the independent full journey). # # Required secrets: BREV_API_KEY + BREV_ORG_ID (or legacy BREV_API_TOKEN), NVIDIA_INFERENCE_API_KEY -# Instance cost: Brev CPU credits (~$0.10/run for 4x16 instance) +# Instance cost: Brev CPU credits; most suites use 4x16, while the performance +# smoke requests 8 vCPU, 32 GB RAM, and 100 GB disk. on: workflow_dispatch: @@ -73,6 +77,7 @@ on: - messaging-providers - messaging-compatible-endpoint - dashboard-remote-bind + - tool-disclosure-performance-smoke - gpu - all use_launchable: @@ -188,7 +193,7 @@ jobs: TEST_SUITE: ${{ inputs.test_suite }} run: | case "$TEST_SUITE" in - full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all) ;; + full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|tool-disclosure-performance-smoke|gpu|all) ;; *) echo "::error::test_suite is not one of the supported Brev E2E suites" exit 1 @@ -213,7 +218,7 @@ jobs: - name: Checkout target branch uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - ref: ${{ env.RESOLVED_BRANCH || inputs.branch || 'main' }} + ref: ${{ env.RESOLVED_BRANCH || inputs.branch || github.ref }} persist-credentials: false - id: tested-ref @@ -267,7 +272,28 @@ jobs: - name: Build CLI run: npm run build:cli - - name: Run ephemeral Brev E2E + - name: Install and verify Brev performance smoke tunnel prerequisite + if: inputs.test_suite == 'tool-disclosure-performance-smoke' + env: + CLOUDFLARED_VERSION: "2026.6.1" + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" + run: | + set -euo pipefail + cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" + curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" + printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - + package="$(dpkg-deb -f "${cloudflared_deb}" Package)" + version="$(dpkg-deb -f "${cloudflared_deb}" Version)" + architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" + if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then + printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 + exit 1 + fi + sudo dpkg -i "${cloudflared_deb}" + cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" + + - id: brev-test + name: Run ephemeral Brev E2E env: NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1" NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE: "1" @@ -277,6 +303,9 @@ jobs: INSTANCE_NAME: ${{ env.BREV_E2E_INSTANCE_NAME }} TEST_SUITE: ${{ inputs.test_suite }} USE_LAUNCHABLE: ${{ inputs.use_launchable && '1' || '0' }} + BREV_MIN_VCPU: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '8' || '' }} + BREV_MIN_RAM: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '32' || '' }} + BREV_MIN_DISK: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '100' || '' }} BREV_PROVIDER: ${{ inputs.brev_provider || vars.BREV_PROVIDER || '' }} BREV_GPU_TYPE: ${{ inputs.brev_gpu_type || vars.BREV_GPU_TYPE || '' }} BREV_GPU_NAME: ${{ inputs.brev_gpu_name || vars.BREV_GPU_NAME || '' }} @@ -286,6 +315,63 @@ jobs: KEEP_ALIVE: ${{ inputs.keep_alive }} run: npx vitest run --project e2e-branch-validation --silent=false --reporter=default + - name: Collect Brev tool-disclosure performance smoke artifacts + if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke' + env: + INSTANCE: ${{ env.BREV_E2E_INSTANCE_NAME }} + TEST_OUTCOME: ${{ steps.brev-test.outcome }} + run: | + set -euo pipefail + artifact_dir="brev-tool-disclosure-performance-smoke-artifacts" + remote_dir="/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts" + archive="${RUNNER_TEMP}/brev-tool-disclosure-performance-smoke-artifacts.tar.gz" + mkdir -p "$artifact_dir" + brev refresh >/dev/null + if ! ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR \ + -o ConnectTimeout=10 "$INSTANCE" \ + "test -d '$remote_dir' && tar -C '$remote_dir' -czf - ." \ + >"$archive"; then + if [ "$TEST_OUTCOME" = "success" ]; then + echo "::error::Successful Brev performance smoke did not produce an artifact archive" + exit 1 + fi + echo "::warning::Failed Brev performance smoke produced no artifact archive" + exit 0 + fi + tar -C "$artifact_dir" -xzf "$archive" + result="$artifact_dir/tool-disclosure-hosted-inference-performance-smoke-completes-one-frozen-task-in-both-modes/tool-disclosure-performance-smoke.json" + if [ ! -f "$result" ]; then + if [ "$TEST_OUTCOME" = "success" ]; then + echo "::error::Successful Brev performance smoke artifact is missing its result JSON" + exit 1 + fi + echo "::warning::Failed Brev performance smoke has only partial artifacts" + fi + + - name: Scan Brev tool-disclosure performance smoke artifacts + if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke' + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + artifact_dir="brev-tool-disclosure-performance-smoke-artifacts" + test -d "$artifact_dir" + if [ -n "${NVIDIA_INFERENCE_API_KEY:-}" ] && \ + grep -R -I -l -F -- "$NVIDIA_INFERENCE_API_KEY" "$artifact_dir" >/dev/null; then + echo "::error::Brev performance smoke artifacts contain an unredacted inference credential" + exit 1 + fi + + - name: Upload Brev tool-disclosure performance smoke artifacts + if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-brev-tool-disclosure-performance-smoke-${{ github.run_attempt }} + path: brev-tool-disclosure-performance-smoke-artifacts/ + if-no-files-found: ignore + include-hidden-files: false + retention-days: 14 + # Collect debugging artifacts from the Brev VM on failure before the # instance gets torn down. Captures the onboard log, sandbox list, # docker state, and gateway status so downstream onboard failures are @@ -306,6 +392,11 @@ jobs: timeout 15s openshell sandbox list > /tmp/nc-debug/sandbox-list.txt 2>&1 timeout 15s openshell gateway status > /tmp/nc-debug/gateway-status.txt 2>&1 timeout 15s docker ps -a > /tmp/nc-debug/docker-ps.txt 2>&1 + free -h > /tmp/nc-debug/memory.txt 2>&1 + df -h > /tmp/nc-debug/disk.txt 2>&1 + timeout 15s sudo systemctl status docker --no-pager > /tmp/nc-debug/docker-service.txt 2>&1 + timeout 15s sudo journalctl -u docker --since "15 minutes ago" --no-pager > /tmp/nc-debug/docker-journal.txt 2>&1 + timeout 15s sudo dmesg | tail -200 > /tmp/nc-debug/kernel-tail.txt 2>&1 timeout 30s tar -C /tmp -czf /tmp/nc-debug.tar.gz nc-debug ' || true scp -o StrictHostKeyChecking=no -o LogLevel=ERROR \ diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 926e44ba42..a258b15756 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -14,7 +14,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." + description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, tool-disclosure-performance-smoke, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected." required: false default: "" type: string @@ -723,6 +723,96 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + tool-disclosure-performance-smoke: + needs: generate-matrix + # This smoke provisions real agent sandboxes and consumes hosted inference. + # Keep it opt-in: it validates performance-test plumbing on ordinary CI hardware, + # but it is not the complete two-campaign performance-test procedure. + if: ${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-performance-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-performance-smoke,') }} + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 60 + env: + E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" + E2E_TARGET_ID: "tool-disclosure-performance-smoke" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-performance-smoke + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_OPENSHELL_CHANNEL: stable + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-ultra + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-ultra + NEMOCLAW_PREFERRED_API: openai-completions + OPENSHELL_GATEWAY: nemoclaw + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - name: Install and verify cloudflared prerequisite + env: + CLOUDFLARED_VERSION: "2026.6.1" + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" + run: | + set -euo pipefail + cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" + curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" + printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - + package="$(dpkg-deb -f "${cloudflared_deb}" Package)" + version="$(dpkg-deb -f "${cloudflared_deb}" Version)" + architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" + if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then + printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 + exit 1 + fi + sudo dpkg -i "${cloudflared_deb}" + cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" + + - name: Install OpenShell CLI + env: + NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" + run: bash scripts/install-openshell.sh + + - name: Run tool-disclosure performance smoke live test + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-live \ + test/e2e/live/tool-disclosure-performance-smoke.test.ts \ + --silent=false --reporter=default + + - name: Upload tool-disclosure performance smoke artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + onboard-negative-paths: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.targets), ',onboard-negative-paths,') }} @@ -4613,6 +4703,7 @@ jobs: openshell-gateway-auth-contract, mcp-bridge, mcp-bridge-dev, + tool-disclosure-performance-smoke, onboard-negative-paths, skill-agent, openclaw-skill-cli, @@ -4718,6 +4809,11 @@ jobs: target: 'mcp-bridge-dev', reason: 'default dispatch excludes moving OpenShell dev artifacts unless explicitly selected', }, + 'tool-disclosure-performance-smoke': { + job: 'tool-disclosure-performance-smoke', + target: 'tool-disclosure-performance-smoke', + reason: 'default dispatch excludes the hosted-inference performance smoke test unless explicitly selected', + }, 'jetson-nvmap-gpu': { job: 'jetson-nvmap-gpu', target: 'jetson-nvmap-gpu', @@ -4848,7 +4944,7 @@ jobs: ? '**Requested jobs:** _(selector rejected by workflow validation)_' : requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` - : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', + : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `tool-disclosure-performance-smoke`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_', `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`, '', '| Job | Result |', diff --git a/agents/langchain-deepagents-code/Dockerfile b/agents/langchain-deepagents-code/Dockerfile index 3b2e62774a..9ef5e26243 100644 --- a/agents/langchain-deepagents-code/Dockerfile +++ b/agents/langchain-deepagents-code/Dockerfile @@ -59,14 +59,16 @@ RUN case "$NEMOCLAW_TOOL_DISCLOSURE" in \ esac # The launcher and startup script read these root-owned files instead of -# trusting process-level environment overrides for inference routing. Invoking -# each launcher validates the build args before the image can complete. +# trusting process-level environment overrides for inference routing or tool +# disclosure. Invoking each launcher validates the build args before the image +# can complete. RUN install -d -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' "$NEMOCLAW_PROXY_HOST" > /usr/local/share/nemoclaw/dcode-proxy-host \ && printf '%s\n' "$NEMOCLAW_PROXY_PORT" > /usr/local/share/nemoclaw/dcode-proxy-port \ + && printf '%s\n' "$NEMOCLAW_TOOL_DISCLOSURE" > /usr/local/share/nemoclaw/dcode-tool-disclosure \ && printf '%s\n' "$NEMOCLAW_INFERENCE_BASE_URL" > /usr/local/share/nemoclaw/dcode-inference-base-url \ - && chown root:root /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-inference-base-url \ - && chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-inference-base-url \ + && chown root:root /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-tool-disclosure /usr/local/share/nemoclaw/dcode-inference-base-url \ + && chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-tool-disclosure /usr/local/share/nemoclaw/dcode-inference-base-url \ && /usr/local/bin/dcode --version \ && /usr/local/bin/dcode.real --version \ && /usr/local/bin/deepagents-code --version diff --git a/agents/langchain-deepagents-code/Dockerfile.base b/agents/langchain-deepagents-code/Dockerfile.base index bc9d9883e2..2c6f41fdee 100644 --- a/agents/langchain-deepagents-code/Dockerfile.base +++ b/agents/langchain-deepagents-code/Dockerfile.base @@ -42,7 +42,7 @@ RUN groupadd -r sandbox \ # Pre-create shell init files for the sandbox user. OpenShell/NemoClaw writes # /tmp/nemoclaw-proxy-env.sh at startup so interactive sessions and dcode share -# the same proxy, CA, inference, HOME, and update-check posture. +# the same proxy, CA, inference, tool-disclosure, HOME, and update-check posture. # hadolint ignore=SC2016 RUN printf '%s\n' \ '# Source runtime proxy + Deep Agents Code config' \ diff --git a/agents/langchain-deepagents-code/dcode-launcher.sh b/agents/langchain-deepagents-code/dcode-launcher.sh index 2e92280e62..76c8001707 100755 --- a/agents/langchain-deepagents-code/dcode-launcher.sh +++ b/agents/langchain-deepagents-code/dcode-launcher.sh @@ -20,6 +20,7 @@ export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sb # dcode no longer uses inference.local. readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host" readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port" +readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure" readonly MANAGED_PROXY_OWNER_UID=0 managed_proxy_file_metadata() { @@ -32,21 +33,21 @@ managed_proxy_file_metadata() { fi } -read_managed_proxy_value() { +read_managed_image_value() { local file="$1" local name="$2" local metadata local value if [ ! -f "$file" ] || [ -L "$file" ] || [ ! -r "$file" ]; then - printf 'Missing or unsafe trusted managed proxy %s file.\n' "$name" >&2 + printf 'Missing or unsafe trusted managed %s file.\n' "$name" >&2 return 1 fi metadata="$(managed_proxy_file_metadata "$file")" || { - printf 'Cannot inspect trusted managed proxy %s file.\n' "$name" >&2 + printf 'Cannot inspect trusted managed %s file.\n' "$name" >&2 return 1 } if [ "$metadata" != "${MANAGED_PROXY_OWNER_UID}:444" ]; then - printf 'Unsafe ownership or mode on trusted managed proxy %s file.\n' "$name" >&2 + printf 'Unsafe ownership or mode on trusted managed %s file.\n' "$name" >&2 return 1 fi value="$(<"$file")" @@ -55,8 +56,9 @@ read_managed_proxy_value() { # Onboard validates the build args and the Dockerfile stores them in root-owned # files. Runtime env is untrusted and cannot override those image-baked values. -PROXY_HOST="$(read_managed_proxy_value "$MANAGED_PROXY_HOST_FILE" "host")" -PROXY_PORT="$(read_managed_proxy_value "$MANAGED_PROXY_PORT_FILE" "port")" +PROXY_HOST="$(read_managed_image_value "$MANAGED_PROXY_HOST_FILE" "proxy host")" +PROXY_PORT="$(read_managed_image_value "$MANAGED_PROXY_PORT_FILE" "proxy port")" +TOOL_DISCLOSURE="$(read_managed_image_value "$MANAGED_TOOL_DISCLOSURE_FILE" "tool-disclosure mode")" unset NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT # Generic proxy fallbacks are outside the managed dcode contract and may carry # host credentials even after the scheme-specific proxy values are normalized. @@ -88,6 +90,14 @@ if ! is_valid_proxy_port "$PROXY_PORT"; then printf '%s\n' 'Invalid NEMOCLAW_PROXY_PORT for the managed runtime proxy.' >&2 exit 1 fi +case "$TOOL_DISCLOSURE" in + progressive | direct) ;; + *) + printf '%s\n' 'Invalid image-baked tool-disclosure mode for the managed runtime.' >&2 + exit 1 + ;; +esac +export NEMOCLAW_TOOL_DISCLOSURE="$TOOL_DISCLOSURE" _PROXY_URL="http://${PROXY_HOST}:${PROXY_PORT}" _NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}" diff --git a/agents/langchain-deepagents-code/managed-dcode-runtime.py b/agents/langchain-deepagents-code/managed-dcode-runtime.py index cf0abf2b5c..f1626b6ae3 100644 --- a/agents/langchain-deepagents-code/managed-dcode-runtime.py +++ b/agents/langchain-deepagents-code/managed-dcode-runtime.py @@ -52,7 +52,13 @@ _MCP_CHILD_BINDING_ENV = "NEMOCLAW_DCODE_MCP_BINDING" _MCP_SEALED_KIND = "sealed-memfd" _MCP_ANONYMOUS_KIND = "anonymous-otmpfile" -_MCP_ANONYMOUS_DIRECTORY = Path("/tmp") +_MCP_ANONYMOUS_DIRECTORIES = (Path("/tmp"), Path("/dev/shm")) +_MCP_ANONYMOUS_DIRECTORY_FALLBACK_ERRNOS = { + errno.EINVAL, + errno.ENOENT, + errno.ENOSYS, + errno.EOPNOTSUPP, +} _MCP_FALLBACK_ERRNOS = { errno.EACCES, errno.EINVAL, @@ -699,13 +705,29 @@ def _sealed_managed_mcp_snapshot(payload: bytes) -> int: raise +def _open_anonymous_managed_mcp_writer(flags: int) -> int: + for index, directory in enumerate(_MCP_ANONYMOUS_DIRECTORIES): + try: + return os.open(directory, flags, 0o600) + except OSError as exc: + is_last = index == len(_MCP_ANONYMOUS_DIRECTORIES) - 1 + if ( + is_last + or exc.errno not in _MCP_ANONYMOUS_DIRECTORY_FALLBACK_ERRNOS + ): + raise + raise RuntimeError( + "managed MCP config requires anonymous O_TMPFILE support" + ) + + def _anonymous_managed_mcp_snapshot(payload: bytes) -> int: writer: int | None = None reader: int | None = None complete = False try: flags = os.O_TMPFILE | os.O_EXCL | os.O_RDWR | os.O_CLOEXEC - writer = os.open(_MCP_ANONYMOUS_DIRECTORY, flags, 0o600) + writer = _open_anonymous_managed_mcp_writer(flags) remaining = memoryview(payload) while remaining: written = os.write(writer, remaining) diff --git a/agents/langchain-deepagents-code/policy-additions.yaml b/agents/langchain-deepagents-code/policy-additions.yaml index 129f4501e1..d0a12c182e 100644 --- a/agents/langchain-deepagents-code/policy-additions.yaml +++ b/agents/langchain-deepagents-code/policy-additions.yaml @@ -24,6 +24,10 @@ filesystem_policy: - /sandbox - /sandbox/.deepagents - /tmp + # OpenShell blocks memfd_create; managed MCP config uses an unlinked, + # read-only O_TMPFILE snapshot on this sandbox-local tmpfs when /tmp's + # backing filesystem does not support anonymous files. + - /dev/shm - /dev/null landlock: diff --git a/agents/langchain-deepagents-code/start.sh b/agents/langchain-deepagents-code/start.sh index 65cd2a0af0..ae19c955e7 100755 --- a/agents/langchain-deepagents-code/start.sh +++ b/agents/langchain-deepagents-code/start.sh @@ -42,6 +42,7 @@ export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference.local/v1}" # or when dcode no longer uses inference.local. readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host" readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port" +readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure" readonly MANAGED_PROXY_OWNER_UID=0 managed_proxy_file_metadata() { @@ -54,31 +55,32 @@ managed_proxy_file_metadata() { fi } -read_managed_proxy_value() { +read_managed_image_value() { local file="$1" local name="$2" local metadata local value if [ ! -f "$file" ] || [ -L "$file" ] || [ ! -r "$file" ]; then - printf 'Missing or unsafe trusted managed proxy %s file.\n' "$name" >&2 + printf 'Missing or unsafe trusted managed %s file.\n' "$name" >&2 return 1 fi metadata="$(managed_proxy_file_metadata "$file")" || { - printf 'Cannot inspect trusted managed proxy %s file.\n' "$name" >&2 + printf 'Cannot inspect trusted managed %s file.\n' "$name" >&2 return 1 } if [ "$metadata" != "${MANAGED_PROXY_OWNER_UID}:444" ]; then - printf 'Unsafe ownership or mode on trusted managed proxy %s file.\n' "$name" >&2 + printf 'Unsafe ownership or mode on trusted managed %s file.\n' "$name" >&2 return 1 fi value="$(<"$file")" printf '%s' "$value" } -# Fail closed if the root-owned image contract is missing. Process-level -# NEMOCLAW_PROXY_* values are not a trusted runtime routing source. -PROXY_HOST="$(read_managed_proxy_value "$MANAGED_PROXY_HOST_FILE" "host")" -PROXY_PORT="$(read_managed_proxy_value "$MANAGED_PROXY_PORT_FILE" "port")" +# Fail closed if the root-owned image contract is missing. Process-level proxy +# and tool-disclosure values are not trusted runtime configuration sources. +PROXY_HOST="$(read_managed_image_value "$MANAGED_PROXY_HOST_FILE" "proxy host")" +PROXY_PORT="$(read_managed_image_value "$MANAGED_PROXY_PORT_FILE" "proxy port")" +TOOL_DISCLOSURE="$(read_managed_image_value "$MANAGED_TOOL_DISCLOSURE_FILE" "tool-disclosure mode")" unset NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT # Generic proxy fallbacks are outside the managed dcode contract and may carry # host credentials even after the scheme-specific proxy values are normalized. @@ -110,6 +112,14 @@ if ! is_valid_proxy_port "$PROXY_PORT"; then printf '%s\n' 'Invalid NEMOCLAW_PROXY_PORT for the managed runtime proxy.' >&2 exit 1 fi +case "$TOOL_DISCLOSURE" in + progressive | direct) ;; + *) + printf '%s\n' 'Invalid image-baked tool-disclosure mode for the managed runtime.' >&2 + exit 1 + ;; +esac +export NEMOCLAW_TOOL_DISCLOSURE="$TOOL_DISCLOSURE" _PROXY_URL="http://${PROXY_HOST}:${PROXY_PORT}" _NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}" @@ -165,6 +175,7 @@ prepare_runtime_env() { # LangSmith values are intentionally excluded because any inherited variable # can be misconfigured with a token and this shared file is readable. write_export_if_set NEMOCLAW_SANDBOX_NAME + write_export_if_set NEMOCLAW_TOOL_DISCLOSURE } >"$tmp" # Dcode intentionally runs as the non-root sandbox user, unlike the # root-supervised OpenClaw/Hermes startup path. This atomic, sandbox-user-owned diff --git a/docs/index.yml b/docs/index.yml index 8eea7f133f..25a1525d97 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -66,6 +66,9 @@ navigation: - page: "Model Capability Audit" path: _build/agent-variants/inference/model-capability-audit.openclaw.generated.mdx slug: model-capability-audit + - page: "Progressive Tool Disclosure Performance Test" + path: _build/agent-variants/inference/progressive-tool-disclosure-performance-test.openclaw.generated.mdx + slug: progressive-tool-disclosure-performance-test - page: "Switch Inference Providers" path: _build/agent-variants/inference/switch-inference-providers.openclaw.generated.mdx slug: switch-inference-providers @@ -249,6 +252,9 @@ navigation: - page: "Model Capability Audit" path: _build/agent-variants/inference/model-capability-audit.hermes.generated.mdx slug: model-capability-audit + - page: "Progressive Tool Disclosure Performance Test" + path: _build/agent-variants/inference/progressive-tool-disclosure-performance-test.hermes.generated.mdx + slug: progressive-tool-disclosure-performance-test - page: "Switch Inference Providers" path: _build/agent-variants/inference/switch-inference-providers.hermes.generated.mdx slug: switch-inference-providers diff --git a/docs/inference/progressive-tool-disclosure-performance-test.mdx b/docs/inference/progressive-tool-disclosure-performance-test.mdx new file mode 100644 index 0000000000..e652bd08ac --- /dev/null +++ b/docs/inference/progressive-tool-disclosure-performance-test.mdx @@ -0,0 +1,327 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Progressive Tool Disclosure Performance Test" +sidebar-title: "Progressive Tool Disclosure Performance Test" +description: "Run the reproducible two-campaign performance test that measures progressive tool disclosure across NemoClaw agent surfaces." +description-agent: "Explains how to prepare, run, and summarize the hardware-neutral progressive tool-disclosure performance test. Use when measuring tool-schema tokens, task success, or latency for OpenClaw, Hermes, and LangChain Deep Agents Code." +keywords: "progressive tool disclosure performance test, hardware-neutral performance test, tool schema tokens, MCP performance test" +position: 5 +--- + +This protocol compares direct tool exposure with progressive disclosure for OpenClaw, Hermes, and LangChain Deep Agents Code. +It produces checksummed evidence and only emits claim text that clears the configured statistical gates. + + +The repository does not include measured results. +Run both campaigns and retain their evidence before you make a performance or quality claim. + + +The explicit-only `tool-disclosure-performance-smoke` E2E job is a smaller operational check for ordinary GitHub-hosted runners. +It runs the route-only acceptance corpus, one frozen task in progressive and direct modes with fresh Deep Agents sandboxes, and a separate routed replay of that task through the request transformer. +The routed replay keeps its recorder on a host-local private Docker bridge within the trusted runner boundary, rejects requests before transformation unless they carry a fresh ephemeral credential, and replaces that credential with the hosted provider credential only on the runner side. +Its output validates live wiring only: it is not a complete campaign, cannot be summarized by this protocol, and does not support performance claims. + +## Exercise the compositional routing experiment + +The performance-test package also contains a test-scoped, independently implemented compositional tool router. +It is an experimental reference path, not a new NemoClaw runtime mode. +Enabling it in tests does not change the meaning of the existing `direct` or `progressive` modes. + +The implementation performs these steps: + +1. Ask an injected decomposer for an ordered list of atomic actions. +2. L2-normalize tool and action embeddings, then perform deterministic exact inner-product retrieval. +3. Merge the first-pass candidates into at most `12` tool-name hints. +4. Ask the decomposer to refine the action boundaries with those hints. +5. Retrieve the top `10` candidates per refined action, select a bounded ordered tool set, and filter only the schemas forwarded in a copied model request. + +The tested request-transform contract leaves the caller-owned executor and policy registry unchanged. +Its integration tests verify that only the copied model request is filtered and that the recording proxy measures the transformed schemas. +This transform is not installed in the frozen direct/progressive campaign. +Malformed decomposition, embedding, or routing results fail open to the original unfiltered request, and the route records counts, safe tool names, fallback class, and timings without retaining prompts or subtask text. + +The model adapters support an OpenAI-compatible chat decomposer and an OpenAI-compatible semantic embeddings endpoint. +Record the exact embedding model and revision for any measured run. +The dependency-free hashing adapter is only for deterministic tests and portable smoke checks; it is lexical and is not a substitute for a recorded semantic retrieval model. + +Run the deterministic routing acceptance suite on an ordinary CPU runner: + +```bash +npx vitest run --project integration \ + test/performance/tool-disclosure-compositional-*.test.ts \ + test/performance/tool-disclosure-recorder.test.ts +``` + +Run the same route-only acceptance corpus with a real OpenAI-compatible decomposer by creating a private configuration file: + +```json +{ + "decomposer": { + "base_url": "https://models.example/v1", + "model": "recorded-decomposer-model", + "revision": "immutable-model-revision", + "api_key_env": "MODEL_API_KEY", + "allow_remote": true, + "reasoning_control": "enable_thinking_false", + "json_object_response": true, + "max_attempts": 2 + }, + "embedding": { + "kind": "portable", + "dimensions": 1024 + }, + "timeout_ms": 120000, + "run_timeout_ms": 900000 +} +``` + +Then run: + +```bash +npm run performance:tool-disclosure -- route-acceptance \ + --output-dir \ + --config +``` + +The command writes `compositional-routing-acceptance.json` and `SHA256SUMS`, exits nonzero when a strict gate fails, and never writes endpoint URLs, credentials, prompts, or decomposed subtask text. +Set `reasoning_control` only when the endpoint supports the matching chat-template extension: `enable_thinking_false` sends `chat_template_kwargs.enable_thinking=false`, while `thinking_false` sends `chat_template_kwargs.thinking=false`. +Set `json_object_response` only when the endpoint supports OpenAI-compatible JSON-object response mode. +Both choices are recorded in the artifact; omit them to use endpoint defaults and prompt-only JSON formatting. +`max_attempts` is the per-decomposition attempt count and must be at most `3`. +`timeout_ms` applies to each remote request and must be at most `300000`; the product of `timeout_ms` and `max_attempts` must not exceed `600000`. +`run_timeout_ms` is the overall route-only deadline across index construction and every corpus case and must be at most `2700000`. +The overall deadline aborts outstanding remote work, exits nonzero, and is recorded with the per-request and retry limits. +Timed-out runs record an explicit execution status, completed-case count, and `run-deadline-exceeded` case evidence instead of attributing unattempted work to a model or embedding failure. +Every attempt contributes to the request and duration counts, and any provider token usage returned for an attempt is aggregated. +The portable embedding mode is a CPU smoke path and is always claim-ineligible. +For a semantic retrieval run, set `embedding.kind` to `openai` and provide its `base_url`, `model`, immutable `revision`, optional `api_key_env`, and remote opt-in. +Remote opt-in sends the acceptance prompts and synthetic tool metadata to the configured HTTPS services; HTTP is accepted only on loopback. + +The route-only corpus contains `20` distinct tools and eight implicit requests spanning zero through five required capabilities. +Its strict gates require every route to complete without fallback, exact decomposition count, decomposition within one step, exact tool recall at `1` and `10`, ordered-chain forwarding, forwarded-schema count, and no-tool behavior. +Passthrough is evaluated as forwarding all `20` schemas, so a failed-open no-tool route cannot be scored as a successful empty route. +The deterministic suite holds decomposition fixed, so passing it proves the router, retriever, transformer, evidence, and fallback contracts; it does not prove that tool hints improve a real model's decomposition. + +For a model-level evaluation, record a paired initial-versus-one-refinement run on the same implicit corpus and report decomposition and embedding usage separately alongside decomposition accuracy, decomposition within one step, exact recall at `1` and `10`, ordered-chain selection, selected-schema count, tokens, and latency. +Then use the existing agent performance test below for execution success, arguments, ordering, schema tokens, and end-to-end latency. +Treat latency as a measurement rather than an acceptance gate because the refinement adds a second decomposition call. + +This experiment routes callable tool schemas in its own synthetic corpus. +It does not implement or claim a dependency-graph planner or compatibility composer. +The route-only command does not execute an agent or a tool; use the full performance procedure below to validate task execution and end-to-end behavior. +It also does not install the router into the frozen direct/progressive campaign. +The opt-in smoke job exercises the transform in one separate, claim-ineligible agent replay and requires successful task execution, the expected tool call, no routing fallback, and fewer model-visible schemas. +That ordinary-runner bridge is test-only; the complete two-campaign procedure keeps the recorder on its documented local host boundary and does not route measurements through the smoke bridge. +A broader agent-level routing comparison must use a separate initial/refined schedule in which both agent cells expose the same complete catalog before the request transformer runs. + +## Understand the frozen protocol + +The preparation command generates one deterministic catalog, task corpus, and paired execution schedule. +Use the generated files without adding, removing, or reordering runs. + +| Phase | Catalog tools | Tasks and repetitions per agent and mode | Purpose | +| --- | ---: | --- | --- | +| `static-visibility` | `16`, `64`, `256`, `512`, and `2209` | One exact capture at each size. | Measure the initial serialized tool-schema bytes, tokens, and visible tool count. | +| `small-control` | `64` | `24` tasks, one repetition. | Check behavior before context pressure becomes large. | +| `primary` | `512` | `24` tasks, five repetitions. | Supply the paired observations used by the reported claim gates. | +| `large-stress` | `2209` | `8` tasks, one repetition. | Exercise the largest generated catalog without widening the primary claim. | + +The schedule contains `942` runs in each campaign and `1884` runs in total. +It keeps each progressive and direct pair adjacent while deterministically balancing which mode runs first. +The primary task set includes single-tool, ordered-chain, near-match, and no-tool cases. + +## Prepare the evidence directory + +Run preparation from a clean NemoClaw checkout at the exact system-under-test revision. +The sandbox base must use an immutable OCI digest; mutable tags such as `latest` are rejected. +Place the evidence directory outside the Git worktree so generated artifacts cannot change the recorded cleanliness state. + +```bash +npm run performance:tool-disclosure -- prepare \ + --output-dir \ + --sandbox-base ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:<64-hex-digest> +``` + +Preparation writes these inputs: + +- `catalog.json` contains the deterministic synthetic tool catalog. +- `primary-tasks.json` and `stress-tasks.json` contain the frozen task definitions. +- `schedule.json` contains both campaigns in execution order. +- `manifest.template.json` contains the shareable evidence schema fields that you must complete. +- The generated OpenClaw fixture directories contain native tool plugins for the scheduled catalog sizes. + +Copy `manifest.template.json` to `manifest.json` and replace every placeholder before the first run. + +```bash +cp /manifest.template.json /manifest.json +``` + +Record the full NemoClaw system-under-test and harness commit SHAs, exact model ID and revision, vLLM image digest and version, serving flags, parser names, OpenShell version, agent versions, accelerator and CPU description, accelerator driver and runtime versions, RAM, and the observed power or performance state. +For a CPU-only run, set `accelerator_count` to `0`, `accelerator_type` to `none`, and the remaining accelerator fields to `not-applicable`. +Keep credentials, endpoint URLs, hostnames, usernames, and local paths out of the manifest. + + +The performance test commands do not provision hardware, start vLLM, or create sandboxes. +Prepare the runtime environment first; the `execute` command starts the loopback recorder and synthetic MCP fixtures, drives the frozen schedule, and writes sanitized raw evidence plus derived run records. + + +## Provision and preflight the performance test host + +In principle, the performance test can run on any hardware with a compatible host environment that can serve the recorded model and complete the frozen schedule. +The required runtime consists of Node.js and a POSIX-compatible shell, an OpenAI-compatible vLLM endpoint with `/metrics` and `/tokenize`, Docker-backed OpenShell sandboxes, `cloudflared` with outbound network access, and working OpenClaw, Hermes, and LangChain Deep Agents Code runtimes. +Provisioning and access procedures are outside the performance test protocol. + +Run host-appropriate preflight checks before starting inference. +The following examples cover common Linux and macOS environments; run only the commands available on the performance test host. + +```bash +uname -a +uname -m +lscpu # Linux CPU inventory +sysctl -n machdep.cpu.brand_string # macOS CPU inventory +free -h # Linux memory inventory +vm_stat # macOS memory inventory +docker version +openshell --version +nemoclaw --version +git rev-parse HEAD +``` + +Accelerator inspection is optional and platform-specific; use the hardware vendor's supported inspection utility when applicable. +Confirm that the host matches the recorded processor and accelerator configuration, has enough available memory for the frozen model, has the required container image by digest, and has no competing workload that could affect the measurements. +Set and record the intended power or performance state before each campaign. +Do not publish raw preflight logs because they can contain hostnames, usernames, or local paths. + +## Route inference through the recorder + +Run one vLLM server and one performance-test recording proxy on the performance test host. +Configure the NemoClaw managed inference route so every model request from every performance-test sandbox passes through the proxy before reaching vLLM. +After vLLM starts, verify its local health endpoint before beginning a campaign. + +```bash +curl --fail --silent --show-error http://127.0.0.1:/health +``` + +```text +OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes + | + NemoClaw managed inference route + | + recording proxy on 127.0.0.1, forwarding /v1/* + | + vLLM + /metrics and /tokenize +``` + +The recorder must bind to `127.0.0.1` and must not be exposed on a non-loopback network interface. +It forwards OpenAI-compatible `/v1` requests without following redirects and records content-free request metadata, including the canonical tool-schema byte count, schema hash, visible tool count, tool names, status, monotonic timing, and run sequence. +It does not retain prompts, messages, headers, request bodies, response bodies, model identifiers, endpoints, or errors from upstream bodies. + +Use vLLM `/tokenize` to count initial serialized tool-schema tokens with the exact served model tokenizer. +Read prompt and generation token counters from vLLM `/metrics` immediately before and after each scheduled run. +Keep performance-test concurrency at `1` so those process-level counter deltas remain attributable to one run. +Use temperature `0`, the manifest's frozen parser configuration, and the same recorded vLLM flags in both modes and both campaigns. + +## Run two fresh campaigns + +Treat the two campaigns as independent executions of the same prepared schedule. +Do not reuse an inference process, sandbox, model session, or recorder event stream across campaigns. + +Copy `execute-config.template.json` outside the evidence directory and fill in the campaign number, loopback vLLM and telemetry URLs, tokenizer model, fixed recorder port, live `vllm_container_name` and `vllm_container_id`, binary overrides when needed, plus `sandbox_names`, `sandbox_container_names`, and live `sandbox_instance_ids` mappings keyed as `::`. +The sandboxes' managed inference route must already target that recorder port. +Do not publish this file because it can contain local routing and sandbox identifiers. +Execution verifies the live vLLM container ID, Docker image ID, model and revision, parser names, public flags, and process start time. +It also verifies each live sandbox status against its expected agent, disclosure mode, immutable image ID, and instance ID. +Only hashes of private container identities and reviewed public configuration enter the evidence bundle. +Each campaign generates a new MCP bearer token and shares it only among that campaign's five short-lived catalog servers; the token is never written to the public evidence artifacts. +The quick-tunnel process disables config-file discovery and receives an isolated temporary `HOME` and `XDG_CONFIG_HOME`, removed when the tunnel closes, so it cannot load the operator's ambient Cloudflare account configuration or credentials. +This attestation path expects vLLM and the OpenShell sandboxes to use the Docker driver so `docker inspect` can verify the live state. + +```bash +npm run performance:tool-disclosure -- execute \ + --output-dir \ + --config +``` + +For campaign 1: + +1. Start a fresh vLLM process from the recorded image digest and flags. +2. Start a fresh loopback recorder and reset its event stream. +3. Create fresh OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes for the scheduled modes and fixtures. +4. Run `execute` for campaign 1. It appends each attempt atomically to `attempt-journal.jsonl`, records hashed live identities in `campaign-attestations.json`, and materializes content-free `raw-events.jsonl` plus one derived `nemoclaw.tool_disclosure_performance_test.v1` row per entry in `runs.jsonl`. +5. Stop vLLM, stop the recorder, and destroy the campaign sandboxes. + +For campaign 2, repeat the same five steps with another fresh vLLM process, recorder event stream, and sandbox set. +Use the same hardware configuration, system-under-test and harness revisions, model revision, image digest, parser configuration, and task artifacts. +If any of those controlled values changes, prepare a new performance test ID and restart both campaigns. + +The agent adapters must use the generated deterministic handlers and verify expected tool names, order, argument hashes, call count, unnecessary calls, and the returned nonce. +Direct and progressive modes must differ only in their disclosure configuration. +Follow the retry allowance in `manifest.json` for setup failures, and do not convert task failures into setup failures. +Once an agent invocation has begun, any incomplete or inconsistent evidence aborts the campaign instead of consuming a setup retry; discard that campaign and restart it with fresh resources. + +## Summarize and verify artifacts + +After all scheduled records are present, run the summarizer in the same evidence directory. + +```bash +npm run performance:tool-disclosure -- summarize --output-dir +``` + +The command consumes `manifest.json` and `runs.jsonl` and writes: + +- `summary.json` with paired aggregates, confidence intervals, and claim-gate decisions. +- `raw-events.jsonl` with content-free measurements from which every run record is re-derived. +- `attempt-journal.jsonl` as the crash-recoverable source for raw and derived run rows. +- `campaign-attestations.json` with vLLM process start times, hashed inference identities and configuration, and hashed live sandbox identities. +- `report.md` with environment metadata, measurements, gate status, and mechanically generated claim text. +- `evidence.json` with the public artifact inventory and SHA-256 values. +- `SHA256SUMS` with the checksum manifest. + +Verify the bundle on the performance test host before you move it. + +```bash +cd +sha256sum -c SHA256SUMS # GNU/Linux +shasum -a 256 -c SHA256SUMS # macOS +``` + +Run the command available on the performance test host. + +Treat every output as untrusted until you review it for credentials, endpoint URLs, hostnames, usernames, local paths, raw prompts, tool arguments, and response text. +Do not hand-edit `summary.json` or the validated claims in `report.md`. +If a correction changes an input or run record, regenerate the summary and checksums. + +## Apply the claim gates + +The summarizer uses a deterministic paired bootstrap with `10,000` samples. +It averages repetitions within each task first, then resamples whole progressive and direct task pairs, so the task is the statistical unit. + +The report can emit each claim only when its gate passes for OpenClaw, Hermes, and LangChain Deep Agents Code in both campaigns at the `512`-tool primary size. + +| Claim | Required gate in every campaign and agent cell | +| --- | --- | +| Initial serialized tool-schema tokens improved. | The paired progressive-minus-direct token difference has a `95%` confidence-interval upper bound below `0`, and the exact static capture has a positive token reduction. | +| Task success is noninferior. | The paired progressive-minus-direct success difference has a `95%` confidence-interval lower bound of at least `-5` percentage points. | +| End-to-end latency improved. | The paired progressive-minus-direct latency difference has a `95%` confidence-interval upper bound below `0`. | + +Repeated static captures must agree exactly on visible tool count, serialized bytes, and tokenizer tokens. +The summarizer excludes setup failures, but scores timeouts, incorrect tool behavior, model errors, and context overflows as task failures. +A blocked gate remains visible in `report.md`, and the report does not emit the corresponding cross-agent claim. + +## Interpret the scope + +The performance test supports claims about the recorded commits, agents, model, serving configuration, hardware configuration, deterministic synthetic catalog, and task corpus. +It does not establish a universal result for other models, hardware, tool distributions, prompts, concurrency levels, or production workloads. +Evaluate another hardware configuration with a separate complete two-campaign run and report that result with its own recorded environment metadata. + +Progressive disclosure is a context optimization, not an authorization boundary. +The OpenClaw fixture measures its native catalog path and does not establish that mcporter-managed MCP schemas are compacted. +The `2209`-tool stress phase supplies boundary evidence but does not widen claims beyond the `512`-tool primary phase. +Two fresh campaigns on one hardware configuration demonstrate repeatability for that configuration, not cross-hardware generality. + +## Related topics + +- [Model Capability Audit](model-capability-audit) explains the evidence fields for model and agent compatibility claims. +- [Tool-Calling Reliability](tool-calling-reliability) separates tool-use behavior from endpoint connectivity failures. diff --git a/package.json b/package.json index 86df0f0dd8..363a256823 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "test:projects:check": "tsx scripts/checks/vitest-project-overlap.ts", "test:titles:check": "tsx scripts/checks/test-title-style.ts", "bench": "tsx scripts/bench/run.ts", + "performance:tool-disclosure": "tsx scripts/performance/tool-disclosure/run.ts", "check": "npx prek run --all-files --stage pre-commit && npx prek run --all-files --stage manual", "check:diff": "npx prek run --from-ref origin/main --to-ref HEAD --stage pre-commit && npx commitlint --from origin/main --to HEAD && npx prek run --from-ref origin/main --to-ref HEAD --stage pre-push", "checks": "tsx scripts/checks/run.ts", diff --git a/scripts/checks/check-cloudflared-update.sh b/scripts/checks/check-cloudflared-update.sh index 3f25aa2fc6..8d99d38a41 100755 --- a/scripts/checks/check-cloudflared-update.sh +++ b/scripts/checks/check-cloudflared-update.sh @@ -2,14 +2,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# invalidState: the three reviewed E2E consumers drift to different cloudflared +# invalidState: the four reviewed E2E consumers drift to different cloudflared # versions/digests, or their shared pin no longer matches the upstream asset. -# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all three +# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all four # workflow pins and independently verifies the downloaded bytes. # whyNotSourceFix: upstream cannot enforce which release NemoClaw workflows use. -# regressionTest: cloudflared-update-check-workflow.test.ts covers three-pin +# regressionTest: cloudflared-update-check-workflow.test.ts covers four-pin # parity, asset URL identity, digest mismatch, and update instructions. -# removalCondition: remove this checker when the three consumers share one +# removalCondition: remove this checker when the four consumers share one # machine-readable dependency manifest with equivalent live asset verification. set -euo pipefail @@ -48,10 +48,10 @@ done < <( "${E2E_WORKFLOW}" ) -[[ "${#version_pins[@]}" -eq 3 ]] \ - || fail "expected exactly three CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}" -[[ "${#sha_pins[@]}" -eq 3 ]] \ - || fail "expected exactly three CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}" +[[ "${#version_pins[@]}" -eq 4 ]] \ + || fail "expected exactly four CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}" +[[ "${#sha_pins[@]}" -eq 4 ]] \ + || fail "expected exactly four CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}" pinned_version="${version_pins[0]}" pinned_sha="$(printf '%s' "${sha_pins[0]}" | tr '[:upper:]' '[:lower:]')" @@ -127,7 +127,7 @@ print_update_instructions() { 'Update locations:' \ " ${workflow_display} CLOUDFLARED_VERSION lines: ${version_lines}" \ " ${workflow_display} CLOUDFLARED_DEB_SHA256 lines: ${sha_lines}" \ - 'Set all three version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2 + 'Set all four version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2 } if [[ "${latest_version}" != "${pinned_version}" ]]; then diff --git a/scripts/performance/README.md b/scripts/performance/README.md new file mode 100644 index 0000000000..3a96a398eb --- /dev/null +++ b/scripts/performance/README.md @@ -0,0 +1,101 @@ + + +# Progressive tool-disclosure performance test + +The progressive tool-disclosure performance test compares direct and +progressive tool visibility for OpenClaw, Hermes, and LangChain Deep Agents +Code against one deterministic synthetic catalog and task corpus. It is the +repeatable comparison harness for this feature. + +In principle, it can run on any hardware with a compatible host environment +that supports the recorded model, Node.js and a POSIX-compatible shell, an +OpenAI-compatible vLLM endpoint with `/metrics` and `/tokenize`, Docker-backed +OpenShell sandboxes, `cloudflared` with outbound network access, and the three +agent runtimes. Results apply to the recorded hardware and configuration; +evaluating another hardware configuration requires a separate complete run. + +Prepare a new evidence directory with the frozen catalog, task sets, schedule, +manifest template, and generated OpenClaw fixtures: + +```bash +npm run performance:tool-disclosure -- prepare \ + --output-dir \ + --sandbox-base ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:<64-hex-digest> +``` + +Run the emitted schedule as two fresh campaigns on the same recorded hardware +and inference configuration. Each campaign requires a fresh vLLM process and +fresh sandboxes. Populate `manifest.json` from `manifest.template.json`, and +append the public-safe run records to `runs.jsonl`. + +Drive each prepared campaign with a private configuration that identifies the +fresh vLLM container and maps the frozen agent/mode/catalog cells to fresh +sandbox names, container names, and instance IDs: + +```bash +npm run performance:tool-disclosure -- execute \ + --output-dir --config +``` + +Execution uses a recoverable `attempt-journal.jsonl`, verifies the live vLLM +image/configuration and Docker-backed sandbox identities, writes only hashed +private identities to the attestation artifact, materializes content-free +`raw-events.jsonl`, and derives `runs.jsonl`; the summarizer rejects records +that cannot be reproduced from the journal and raw evidence. + +Summarize the completed evidence set: + +```bash +npm run performance:tool-disclosure -- summarize --output-dir +``` + +The summarizer writes `summary.json`, `report.md`, `evidence.json`, and +`SHA256SUMS`. It derives public claim text mechanically and leaves claims +blocked unless the required confidence-interval gate passes for every agent in +both campaigns. + +The same package includes a test-scoped independent compositional tool routing +experiment. It runs two-pass atomic decomposition, normalized exact +inner-product retrieval, and a bounded tool-hint refinement. Integration tests +exercise its request-transform contract through the recorder while leaving the +caller-owned executor registry unchanged. The transform is not enabled in the +frozen direct/progressive campaign. Run its deterministic CPU acceptance suite +with: + +```bash +npx vitest run --project integration \ + test/performance/tool-disclosure-compositional-*.test.ts \ + test/performance/tool-disclosure-recorder.test.ts +``` + +Those tests prove routing mechanics and strict route-quality gates with frozen +decomposition inputs. They do not establish a model-level decomposition +improvement. The full protocol explains the separate paired initial/refined +evidence required for that result. + +The explicit live smoke also runs a separate routed replay after its frozen +direct/progressive cells. That replay must preserve task correctness and the +expected tool call while reducing model-visible schemas without fallback. It +does not alter either frozen cell and remains claim-ineligible. On an ordinary +GitHub runner, its recorder uses an authenticated, host-local private Docker +bridge inside the trusted runner boundary. Its ephemeral ingress credential is +replaced before upstream forwarding; full campaigns retain the protocol's +local recorder topology. + +Run the route-only corpus with a real decomposer using: + +```bash +npm run performance:tool-disclosure -- route-acceptance \ + --output-dir \ + --config +``` + +The protocol documents the private config, semantic and portable embedding +options, public-safe output, remote-content boundary, and strict exit gates. + +See the [progressive tool-disclosure performance-test protocol](../../docs/inference/progressive-tool-disclosure-performance-test.mdx) +for the hardware-neutral workflow, recorder topology, artifact rules, claim +gates, and limitations. diff --git a/scripts/performance/tool-disclosure/artifacts.ts b/scripts/performance/tool-disclosure/artifacts.ts new file mode 100644 index 0000000000..a3ed0bc4ce --- /dev/null +++ b/scripts/performance/tool-disclosure/artifacts.ts @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +const SAFE_ARTIFACT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u; + +export function ensureCampaignDirectory(outputDir: string, resume: boolean): string { + const resolved = path.resolve(outputDir); + if (fs.existsSync(resolved)) { + const stat = fs.lstatSync(resolved); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error("performance test output path must be a real directory"); + } + const entries = fs.readdirSync(resolved); + if (!resume && entries.length > 0) { + throw new Error("performance test output directory is not empty; pass --resume to reuse it"); + } + } else { + fs.mkdirSync(resolved, { recursive: true, mode: 0o700 }); + } + return resolved; +} + +export function artifactPath(outputDir: string, name: string): string { + if (!SAFE_ARTIFACT_NAME.test(name)) + throw new Error(`invalid performance test artifact name: ${name}`); + const root = path.resolve(outputDir); + const candidate = path.resolve(root, name); + if (path.dirname(candidate) !== root) + throw new Error("performance test artifact escaped output directory"); + if (fs.existsSync(candidate) && fs.lstatSync(candidate).isSymbolicLink()) { + throw new Error(`performance test artifact must not be a symlink: ${name}`); + } + return candidate; +} + +export function writeJsonArtifact(outputDir: string, name: string, value: unknown): string { + const destination = artifactPath(outputDir, name); + const temporary = `${destination}.${process.pid}.${Date.now()}.tmp`; + fs.writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + fs.renameSync(temporary, destination); + return destination; +} + +export function writeTextArtifact(outputDir: string, name: string, value: string): string { + const destination = artifactPath(outputDir, name); + const temporary = `${destination}.${process.pid}.${Date.now()}.tmp`; + fs.writeFileSync(temporary, value, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + fs.renameSync(temporary, destination); + return destination; +} + +export function appendJsonLine(outputDir: string, name: string, value: unknown): void { + const destination = artifactPath(outputDir, name); + fs.appendFileSync(destination, `${JSON.stringify(value)}\n`, { + encoding: "utf8", + mode: 0o600, + }); +} + +export function readJsonLines(filePath: string): T[] { + const raw = fs.readFileSync(filePath, "utf8"); + return raw + .split(/\r?\n/u) + .filter((line) => line.trim().length > 0) + .map((line, index) => { + try { + return JSON.parse(line) as T; + } catch { + throw new Error(`invalid JSONL at line ${index + 1}`); + } + }); +} + +export function sha256File(filePath: string): string { + return createHash("sha256").update(fs.readFileSync(filePath)).digest("hex"); +} + +export function writeChecksumManifest(outputDir: string, names: readonly string[]): string { + const unique = [...new Set(names)].sort(); + const lines = unique.map((name) => { + const file = artifactPath(outputDir, name); + if (!fs.statSync(file).isFile()) throw new Error(`cannot checksum non-file artifact: ${name}`); + return `${sha256File(file)} ${name}`; + }); + const destination = artifactPath(outputDir, "SHA256SUMS"); + fs.writeFileSync(destination, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); + return destination; +} + +export function scanArtifactsForForbiddenValues( + outputDir: string, + names: readonly string[], + forbiddenValues: readonly string[], +): void { + const forbidden = forbiddenValues.filter((value) => value.length > 0); + for (const name of names) { + const content = fs.readFileSync(artifactPath(outputDir, name), "utf8"); + const leaked = forbidden.find((value) => content.includes(value)); + if (leaked) throw new Error(`performance test artifact ${name} contains a forbidden value`); + } +} diff --git a/scripts/performance/tool-disclosure/assemble-run.ts b/scripts/performance/tool-disclosure/assemble-run.ts new file mode 100644 index 0000000000..c43c399e30 --- /dev/null +++ b/scripts/performance/tool-disclosure/assemble-run.ts @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { gradeTaskRun, type RecordedSyntheticCall } from "./grading"; +import type { ToolDisclosureRecordingEvent } from "./recorder"; +import type { ScheduledToolDisclosureRun } from "./schedule"; +import type { SyntheticPerformanceTask } from "./tasks"; +import { + type PerformanceTestRunOutcome, + type RunCorrectness, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + type ToolDisclosureManifest, + type ToolDisclosureRun, +} from "./types"; + +export interface RunInvocationResult { + exit_code: number | null; + timed_out: boolean; + elapsed_ms: number; + final_output: string; +} + +function successfulStaticCorrectness(): RunCorrectness { + return { + task_success: true, + expected_tool_names: true, + expected_tool_order: true, + expected_arguments: true, + expected_call_count: true, + nonce_present: true, + unnecessary_tool_calls: 0, + }; +} + +function failedCorrectness(callCount: number): RunCorrectness { + return { + task_success: false, + expected_tool_names: false, + expected_tool_order: false, + expected_arguments: false, + expected_call_count: false, + nonce_present: false, + unnecessary_tool_calls: callCount, + }; +} + +/** + * Convert transient model output plus content-free call/recorder events into a + * public-safe run record. Raw output and tool arguments never cross this API. + */ +export function assembleToolDisclosureRun(options: { + manifest: ToolDisclosureManifest; + scheduled: ScheduledToolDisclosureRun; + task?: SyntheticPerformanceTask; + calls: readonly RecordedSyntheticCall[]; + recorderEvents: readonly ToolDisclosureRecordingEvent[]; + invocation: RunInvocationResult; + initialSchemaTokens: number; + promptTokens?: number; + completionTokens?: number; + failureOutcome?: Exclude; +}): ToolDisclosureRun { + if (!Number.isSafeInteger(options.initialSchemaTokens) || options.initialSchemaTokens < 0) { + throw new Error("initial schema token count must be a non-negative integer"); + } + const scheduledCampaign = `campaign-${options.scheduled.campaign}`; + if (!options.manifest.campaigns.some((item) => item.campaign_id === scheduledCampaign)) { + throw new Error(`scheduled campaign ${scheduledCampaign} is absent from the manifest`); + } + if (options.scheduled.phase !== "static-visibility" && !options.task) { + throw new Error("task runs require their frozen task definition"); + } + if (options.task && options.scheduled.task_id !== options.task.id) { + throw new Error("scheduled task does not match the supplied task definition"); + } + + const modelEvents = options.recorderEvents.filter((event) => event.model_call_sequence !== null); + const initial = modelEvents.find((event) => event.model_call_sequence === 1); + let outcome: PerformanceTestRunOutcome; + let correctness: RunCorrectness; + if (options.failureOutcome) { + outcome = options.failureOutcome; + correctness = failedCorrectness(options.calls.length); + } else if (options.invocation.timed_out) { + outcome = "timeout"; + correctness = failedCorrectness(options.calls.length); + } else if (options.invocation.exit_code !== 0 || !initial || initial.outcome !== "completed") { + outcome = "model-error"; + correctness = failedCorrectness(options.calls.length); + } else if (options.scheduled.phase === "static-visibility") { + outcome = options.initialSchemaTokens > 0 ? "success" : "model-error"; + correctness = + options.initialSchemaTokens > 0 + ? successfulStaticCorrectness() + : failedCorrectness(options.calls.length); + } else { + const graded = gradeTaskRun( + options.task as SyntheticPerformanceTask, + options.calls, + options.invocation.final_output, + ); + outcome = graded.outcome; + correctness = graded.correctness; + } + + return { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: options.manifest.performance_test_id, + campaign_id: scheduledCampaign, + run_id: options.scheduled.run_id, + phase: options.scheduled.phase, + agent: options.scheduled.agent, + mode: options.scheduled.mode, + catalog_size: options.scheduled.catalog_size, + task_id: options.task?.id ?? "static-capture", + ...(options.task ? { task_kind: options.task.kind } : {}), + repetition: options.scheduled.repetition, + execution_seed: options.manifest.protocol.execution_seed, + outcome, + scored: options.scheduled.phase !== "static-visibility" && outcome !== "setup-error", + correctness, + measurements: { + initial_tool_schema: { + tool_count: initial?.visible_tool_count ?? 0, + serialized_bytes: initial?.canonical_tools_json_bytes ?? 0, + tokenizer_tokens: options.initialSchemaTokens, + }, + ...(options.promptTokens === undefined ? {} : { total_prompt_tokens: options.promptTokens }), + ...(options.completionTokens === undefined + ? {} + : { completion_tokens: options.completionTokens }), + ...(initial?.streaming !== true || + initial.time_to_first_byte_ms === null || + initial.time_to_first_byte_ms === undefined + ? {} + : { time_to_first_response_byte_ms: initial.time_to_first_byte_ms }), + inference_time_ms: modelEvents.reduce((total, event) => total + event.duration_ms, 0), + end_to_end_time_ms: options.invocation.elapsed_ms, + model_calls: modelEvents.length, + discovery_calls: Math.max(0, modelEvents.length - options.calls.length - 1), + }, + }; +} diff --git a/scripts/performance/tool-disclosure/catalog.ts b/scripts/performance/tool-disclosure/catalog.ts new file mode 100644 index 0000000000..f55429cca7 --- /dev/null +++ b/scripts/performance/tool-disclosure/catalog.ts @@ -0,0 +1,735 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +export const TOOL_DISCLOSURE_PERFORMANCE_TEST_CATALOG_SCHEMA_VERSION = + "nemoclaw.tool_disclosure.performance_test.catalog.v1" as const; +export const DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED = + "nemoclaw-tool-disclosure-performance-test-v1"; +export const MAX_SYNTHETIC_TOOLS = 2_209; + +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; +export interface JsonObject { + [key: string]: JsonValue; +} + +export interface JsonSchema { + type: "object" | "array" | "string" | "integer" | "number" | "boolean"; + description?: string; + properties?: Readonly>; + required?: readonly string[]; + additionalProperties?: boolean; + items?: JsonSchema; + enum?: readonly JsonPrimitive[]; + minLength?: number; + maxLength?: number; + pattern?: string; + minimum?: number; + maximum?: number; + minItems?: number; + maxItems?: number; + uniqueItems?: boolean; +} + +export interface OpenAIFunctionTool { + type: "function"; + function: { + name: string; + description: string; + parameters: JsonSchema; + }; +} + +/** + * The categories are deliberately synthetic and vendor-neutral. Their labels, + * resources, and operations make every generated tool understandable without + * implying access to a real service or copying a production API. + */ +export const SYNTHETIC_CATEGORIES = [ + { + id: "calendar", + label: "calendar", + resource: "event", + purpose: "appointments, availability, and event logistics", + operations: [ + "find", + "schedule", + "reschedule", + "cancel", + "list_attendees", + "check_availability", + ], + }, + { + id: "contacts", + label: "contacts", + resource: "contact", + purpose: "people and organization directory records", + operations: ["find", "create", "update", "merge", "list_groups", "verify"], + }, + { + id: "email", + label: "email", + resource: "message", + purpose: "mailbox messages and threads", + operations: ["search", "draft", "send", "archive", "summarize", "list_attachments"], + }, + { + id: "documents", + label: "documents", + resource: "document", + purpose: "text documents and their revisions", + operations: ["search", "create", "update", "compare", "export", "list_revisions"], + }, + { + id: "storage", + label: "storage", + resource: "file", + purpose: "files, folders, and object metadata", + operations: ["find", "upload", "copy", "move", "share", "inspect_metadata"], + }, + { + id: "messaging", + label: "messaging", + resource: "conversation", + purpose: "chat conversations and channel messages", + operations: ["search", "post", "reply", "list_members", "summarize", "pin"], + }, + { + id: "projects", + label: "projects", + resource: "project", + purpose: "project plans, milestones, and ownership", + operations: ["find", "create", "update", "list_milestones", "assign_owner", "summarize"], + }, + { + id: "issues", + label: "issues", + resource: "issue", + purpose: "work items, defects, and status tracking", + operations: ["search", "create", "update", "link", "list_blockers", "triage"], + }, + { + id: "source_control", + label: "source control", + resource: "revision", + purpose: "repositories, revisions, and code review metadata", + operations: ["search", "inspect", "compare", "list_changes", "list_reviews", "summarize"], + }, + { + id: "ci_cd", + label: "continuous integration", + resource: "pipeline", + purpose: "build pipelines, jobs, and deployment runs", + operations: ["find", "inspect", "retry", "cancel", "list_jobs", "summarize"], + }, + { + id: "observability", + label: "observability", + resource: "signal", + purpose: "metrics, logs, traces, and service health", + operations: ["query", "inspect", "compare", "list_alerts", "summarize", "correlate"], + }, + { + id: "incidents", + label: "incidents", + resource: "incident", + purpose: "operational incidents and response coordination", + operations: ["find", "create", "update", "list_responders", "summarize", "close"], + }, + { + id: "infrastructure", + label: "infrastructure", + resource: "resource", + purpose: "deployed infrastructure and configuration state", + operations: ["find", "inspect", "compare", "list_dependencies", "validate", "summarize"], + }, + { + id: "compute", + label: "compute", + resource: "compute allocation", + purpose: "compute capacity, allocations, and runtime state", + operations: ["find", "inspect", "list_capacity", "compare", "validate", "summarize"], + }, + { + id: "databases", + label: "databases", + resource: "dataset", + purpose: "database objects, queries, and metadata", + operations: ["find", "describe", "query", "list_columns", "validate", "summarize"], + }, + { + id: "analytics", + label: "analytics", + resource: "analysis", + purpose: "reports, measures, and analytical results", + operations: ["find", "calculate", "compare", "list_dimensions", "forecast", "summarize"], + }, + { + id: "finance", + label: "finance", + resource: "financial record", + purpose: "budgets, transactions, and financial summaries", + operations: ["find", "calculate", "compare", "list_transactions", "validate", "summarize"], + }, + { + id: "commerce", + label: "commerce", + resource: "order", + purpose: "orders, products, and fulfillment records", + operations: ["find", "create", "update", "list_items", "track", "summarize"], + }, + { + id: "travel", + label: "travel", + resource: "itinerary", + purpose: "bookings, itineraries, and travel options", + operations: ["search", "inspect", "compare", "list_segments", "validate", "summarize"], + }, + { + id: "maps", + label: "maps", + resource: "place", + purpose: "places, routes, and geographic information", + operations: ["search", "inspect", "route", "list_nearby", "compare", "summarize"], + }, + { + id: "weather", + label: "weather", + resource: "forecast", + purpose: "forecasts, conditions, and weather alerts", + operations: ["find", "inspect", "compare", "list_alerts", "validate", "summarize"], + }, + { + id: "research", + label: "research", + resource: "source", + purpose: "research sources, evidence, and citations", + operations: ["search", "inspect", "compare", "list_citations", "validate", "summarize"], + }, + { + id: "knowledge", + label: "knowledge", + resource: "knowledge entry", + purpose: "knowledge-base entries and linked concepts", + operations: ["search", "inspect", "create", "update", "list_links", "summarize"], + }, + { + id: "media", + label: "media", + resource: "media asset", + purpose: "audio, image, and video asset metadata", + operations: ["search", "inspect", "transcode", "list_variants", "compare", "summarize"], + }, +] as const; + +export type SyntheticCategoryId = (typeof SYNTHETIC_CATEGORIES)[number]["id"]; +export type SchemaComplexity = "small" | "medium" | "large"; + +export interface SyntheticTool { + index: number; + category: SyntheticCategoryId; + operation: string; + complexity: SchemaComplexity; + definition: OpenAIFunctionTool; + handler: { + kind: "deterministic_fixture_v1"; + key: string; + }; +} + +export interface SyntheticCatalog { + schema_version: typeof TOOL_DISCLOSURE_PERFORMANCE_TEST_CATALOG_SCHEMA_VERSION; + seed: string; + size: number; + max_size: typeof MAX_SYNTHETIC_TOOLS; + tools_sha256: string; + tools: SyntheticTool[]; +} + +export interface SyntheticCatalogOptions { + size?: number; + seed?: string | number; +} + +export interface SyntheticToolResult extends JsonObject { + ok: true; + tool_name: string; + category: SyntheticCategoryId; + operation: string; + nonce: string; + arguments_sha256: string; + summary: string; +} + +export class SyntheticToolInvocationError extends Error {} + +const TOOL_NAME_PATTERN = /^[A-Za-z0-9_-]{1,64}$/; +// A 25/50/25 cycle makes the medium schema the catalog median while retaining +// substantial small and large populations for scaling comparisons. The medium +// definition is intentionally near the performance test's roughly 400-token target; +// authoritative token counts still come from the pinned model tokenizer. +const COMPLEXITY_CYCLE: readonly SchemaComplexity[] = [ + "small", + "small", + "small", + "small", + "small", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "medium", + "large", + "large", + "large", + "large", + "large", +]; + +export function canonicalJson(value: JsonValue): string { + return canonicalize(value, new Set()); +} + +function canonicalize(value: JsonValue, ancestors: Set): string { + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new TypeError("canonical JSON does not support non-finite numbers"); + return JSON.stringify(value); + } + if (typeof value !== "object") { + throw new TypeError(`canonical JSON does not support ${typeof value}`); + } + if (ancestors.has(value)) throw new TypeError("canonical JSON does not support cyclic values"); + + ancestors.add(value); + try { + if (Array.isArray(value)) { + return `[${value.map((item) => canonicalize(item, ancestors)).join(",")}]`; + } + const members = Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalize(value[key], ancestors)}`); + return `{${members.join(",")}}`; + } finally { + ancestors.delete(value); + } +} + +export function sha256Hex(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +function normalizeSeed(seed: string | number): string { + if (typeof seed === "number" && (!Number.isSafeInteger(seed) || seed < 0)) { + throw new TypeError("catalog seed numbers must be non-negative safe integers"); + } + const normalized = String(seed); + if (normalized.length === 0) throw new TypeError("catalog seed must not be empty"); + return normalized; +} + +function hashUint32(seed: string, scope: string): number { + const digest = createHash("sha256").update(`${seed}\0${scope}`, "utf8").digest(); + return digest.readUInt32BE(0); +} + +function shuffledCategories(seed: string): readonly (typeof SYNTHETIC_CATEGORIES)[number][] { + const categories = [...SYNTHETIC_CATEGORIES]; + for (let index = categories.length - 1; index > 0; index -= 1) { + const swap = hashUint32(seed, `category-permutation:${index}`) % (index + 1); + [categories[index], categories[swap]] = [categories[swap], categories[index]]; + } + return categories; +} + +function complexityForIndex(index: number): SchemaComplexity { + return COMPLEXITY_CYCLE[index % COMPLEXITY_CYCLE.length]; +} + +function resourceIdSchema(resource: string): JsonSchema { + return { + type: "string", + description: `Stable synthetic ${resource} identifier. A prior tool nonce is also valid.`, + minLength: 1, + maxLength: 96, + pattern: "^[A-Za-z0-9][A-Za-z0-9_.:-]*$", + }; +} + +function buildParameters( + category: (typeof SYNTHETIC_CATEGORIES)[number], + operation: string, + complexity: SchemaComplexity, +): JsonSchema { + const properties: Record = { + resource_id: resourceIdSchema(category.resource), + query: { + type: "string", + description: `Precise synthetic request for the ${operation} operation.`, + minLength: 1, + maxLength: 160, + }, + }; + const required = ["resource_id", "query"]; + + if (complexity !== "small") { + properties.limit = { + type: "integer", + description: "Maximum number of deterministic fixture records to consider.", + minimum: 1, + maximum: 50, + }; + properties.include_archived = { + type: "boolean", + description: "Whether archived synthetic records participate in the operation.", + }; + properties.tags = { + type: "array", + description: "Optional synthetic labels used to narrow the operation.", + items: { type: "string", minLength: 1, maxLength: 32 }, + maxItems: 5, + uniqueItems: true, + }; + required.push("limit"); + } + + if (complexity === "large") { + properties.filters = { + type: "object", + description: `Structured filters for the synthetic ${category.resource}.`, + properties: { + status: { + type: "string", + enum: ["active", "pending", "complete"], + description: "Synthetic lifecycle state to select.", + }, + owner: { + type: "string", + minLength: 1, + maxLength: 48, + description: "Synthetic owner identifier.", + }, + window: { + type: "object", + description: "Inclusive logical-day range within the fixture timeline.", + properties: { + start_day: { type: "integer", minimum: 1, maximum: 365 }, + end_day: { type: "integer", minimum: 1, maximum: 365 }, + }, + required: ["start_day", "end_day"], + additionalProperties: false, + }, + }, + required: ["status", "owner", "window"], + additionalProperties: false, + }; + properties.options = { + type: "object", + description: "Deterministic projection and ordering controls.", + properties: { + sort_order: { type: "string", enum: ["ascending", "descending"] }, + fields: { + type: "array", + items: { type: "string", minLength: 1, maxLength: 32 }, + minItems: 1, + maxItems: 6, + uniqueItems: true, + }, + explain: { type: "boolean" }, + }, + required: ["sort_order", "fields", "explain"], + additionalProperties: false, + }; + properties.correlation_id = { + type: "string", + description: "Caller-provided identifier for deterministic chain assertions.", + minLength: 1, + maxLength: 64, + pattern: "^[A-Za-z0-9_.:-]+$", + }; + required.push("filters", "options"); + } + + return { + type: "object", + description: `Inputs for a synthetic ${category.label} performance test operation.`, + properties, + required, + additionalProperties: false, + }; +} + +function generateTool( + index: number, + seed: string, + categories: readonly (typeof SYNTHETIC_CATEGORIES)[number][], +): SyntheticTool { + const category = categories[index % categories.length]; + const occurrence = Math.floor(index / categories.length); + const operationOffset = + hashUint32(seed, `operation-offset:${category.id}`) % category.operations.length; + const operation = + category.operations[(occurrence + operationOffset) % category.operations.length]; + const complexity = complexityForIndex(index); + const name = `performance_test_${category.id}_${operation}_${index.toString().padStart(4, "0")}`; + + if (!TOOL_NAME_PATTERN.test(name)) throw new Error(`generated unsafe tool name: ${name}`); + + return { + index, + category: category.id, + operation, + complexity, + definition: { + type: "function", + function: { + name, + description: [ + `Synthetic ${category.label} fixture: ${operation.replaceAll("_", " ")} one ${category.resource}.`, + `Use for performance test requests about ${category.purpose}; it never contacts a real service.`, + ].join(" "), + parameters: buildParameters(category, operation, complexity), + }, + }, + handler: { + kind: "deterministic_fixture_v1", + key: sha256Hex(`${seed}\0handler\0${index}`).slice(0, 24), + }, + }; +} + +function assertCatalogSize(size: number): void { + if (!Number.isInteger(size) || size < 1 || size > MAX_SYNTHETIC_TOOLS) { + throw new RangeError(`catalog size must be an integer from 1 through ${MAX_SYNTHETIC_TOOLS}`); + } +} + +export function generateSyntheticTools( + size = MAX_SYNTHETIC_TOOLS, + seed: string | number = DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, +): SyntheticTool[] { + assertCatalogSize(size); + const normalizedSeed = normalizeSeed(seed); + const categories = shuffledCategories(normalizedSeed); + return Array.from({ length: size }, (_, index) => + generateTool(index, normalizedSeed, categories), + ); +} + +export function generateSyntheticCatalog(options: SyntheticCatalogOptions = {}): SyntheticCatalog { + const size = options.size ?? MAX_SYNTHETIC_TOOLS; + const seed = normalizeSeed(options.seed ?? DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED); + const tools = generateSyntheticTools(size, seed); + return { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_CATALOG_SCHEMA_VERSION, + seed, + size, + max_size: MAX_SYNTHETIC_TOOLS, + tools_sha256: sha256Hex(canonicalJson(tools as unknown as JsonValue)), + tools, + }; +} + +export function generateCatalogPrefix(catalog: SyntheticCatalog, size: number): SyntheticCatalog { + assertCatalogSize(size); + if (size > catalog.size) { + throw new RangeError(`catalog prefix ${size} exceeds source catalog size ${catalog.size}`); + } + const tools = catalog.tools.slice(0, size); + return { + ...catalog, + size, + tools_sha256: sha256Hex(canonicalJson(tools as unknown as JsonValue)), + tools, + }; +} + +export function generateCatalogPrefixes( + catalog: SyntheticCatalog, + sizes: readonly number[], +): SyntheticCatalog[] { + return sizes.map((size) => generateCatalogPrefix(catalog, size)); +} + +export function toOpenAIFunctionTools(catalog: SyntheticCatalog): OpenAIFunctionTool[] { + return catalog.tools.map((tool) => tool.definition); +} + +export function buildSyntheticArguments( + tool: SyntheticTool, + variant = 0, + overrides: JsonObject = {}, +): JsonObject { + if (!Number.isSafeInteger(variant) || variant < 0) { + throw new TypeError("argument variant must be a non-negative safe integer"); + } + const argumentsObject: JsonObject = { + resource_id: `${tool.category}-${tool.index}-${variant}`, + query: `${tool.operation.replaceAll("_", " ")} fixture request ${variant}`, + }; + if (tool.complexity !== "small") { + argumentsObject.limit = 3 + (variant % 5); + argumentsObject.include_archived = variant % 2 === 0; + argumentsObject.tags = [`fixture-${variant % 7}`, tool.category]; + } + if (tool.complexity === "large") { + const startDay = 1 + (variant % 300); + argumentsObject.filters = { + status: (["active", "pending", "complete"] as const)[variant % 3], + owner: `owner-${variant % 11}`, + window: { start_day: startDay, end_day: startDay + 7 }, + }; + argumentsObject.options = { + sort_order: variant % 2 === 0 ? "ascending" : "descending", + fields: ["id", "status", "updated_at"], + explain: true, + }; + argumentsObject.correlation_id = `correlation-${tool.index}-${variant}`; + } + return { ...argumentsObject, ...overrides }; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function validateAgainstSchema(value: unknown, schema: JsonSchema, path: string): string[] { + if (schema.type === "object") { + if (!isPlainObject(value)) return [`${path} must be an object`]; + const properties = schema.properties ?? {}; + const errors: string[] = []; + for (const required of schema.required ?? []) { + if (!(required in value)) errors.push(`${path}.${required} is required`); + } + if (schema.additionalProperties === false) { + for (const key of Object.keys(value)) { + if (!(key in properties)) errors.push(`${path}.${key} is not allowed`); + } + } + for (const [key, child] of Object.entries(properties)) { + if (key in value) errors.push(...validateAgainstSchema(value[key], child, `${path}.${key}`)); + } + return errors; + } + if (schema.type === "array") { + if (!Array.isArray(value)) return [`${path} must be an array`]; + const errors: string[] = []; + if (schema.minItems !== undefined && value.length < schema.minItems) { + errors.push(`${path} must contain at least ${schema.minItems} item(s)`); + } + if (schema.maxItems !== undefined && value.length > schema.maxItems) { + errors.push(`${path} must contain at most ${schema.maxItems} item(s)`); + } + if ( + schema.uniqueItems && + new Set(value.map((item) => canonicalJson(item as JsonValue))).size !== value.length + ) { + errors.push(`${path} items must be unique`); + } + if (schema.items) { + value.forEach((item, index) => { + errors.push( + ...validateAgainstSchema(item, schema.items as JsonSchema, `${path}[${index}]`), + ); + }); + } + return errors; + } + if (schema.type === "string") { + if (typeof value !== "string") return [`${path} must be a string`]; + const errors: string[] = []; + if (schema.minLength !== undefined && value.length < schema.minLength) + errors.push(`${path} is too short`); + if (schema.maxLength !== undefined && value.length > schema.maxLength) + errors.push(`${path} is too long`); + if (schema.pattern && !new RegExp(schema.pattern).test(value)) + errors.push(`${path} has an invalid format`); + if (schema.enum && !schema.enum.includes(value)) + errors.push(`${path} must be an allowed value`); + return errors; + } + if (schema.type === "boolean") + return typeof value === "boolean" ? [] : [`${path} must be a boolean`]; + if (schema.type === "integer" || schema.type === "number") { + if (typeof value !== "number" || !Number.isFinite(value)) return [`${path} must be a number`]; + const errors: string[] = []; + if (schema.type === "integer" && !Number.isInteger(value)) + errors.push(`${path} must be an integer`); + if (schema.minimum !== undefined && value < schema.minimum) + errors.push(`${path} is below its minimum`); + if (schema.maximum !== undefined && value > schema.maximum) + errors.push(`${path} is above its maximum`); + return errors; + } + return [`${path} uses an unsupported schema type`]; +} + +export function validateSyntheticArguments(tool: SyntheticTool, args: unknown): string[] { + return validateAgainstSchema(args, tool.definition.function.parameters, "arguments"); +} + +export function syntheticResultNonce(tool: SyntheticTool, args: JsonObject): string { + const errors = validateSyntheticArguments(tool, args); + if (errors.length > 0) throw new SyntheticToolInvocationError(errors.join("; ")); + return `nonce_${sha256Hex(`${tool.handler.key}\0${canonicalJson(args)}`).slice(0, 20)}`; +} + +export function executeSyntheticTool(tool: SyntheticTool, args: JsonObject): SyntheticToolResult { + const argumentsSha256 = sha256Hex(canonicalJson(args)); + return { + ok: true, + tool_name: tool.definition.function.name, + category: tool.category, + operation: tool.operation, + nonce: syntheticResultNonce(tool, args), + arguments_sha256: argumentsSha256, + summary: `${tool.operation.replaceAll("_", " ")} completed for ${String(args.resource_id)}`, + }; +} + +export function validateSyntheticCatalog(catalog: SyntheticCatalog): string[] { + const errors: string[] = []; + if (catalog.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_CATALOG_SCHEMA_VERSION) { + errors.push(`unsupported catalog schema version: ${catalog.schema_version}`); + } + if (catalog.size !== catalog.tools.length) errors.push("catalog size does not match tool count"); + if (catalog.max_size !== MAX_SYNTHETIC_TOOLS) errors.push("catalog max_size is not canonical"); + + const names = new Set(); + catalog.tools.forEach((tool, index) => { + const name = tool.definition.function.name; + if (tool.index !== index) + errors.push(`tool at position ${index} has non-contiguous index ${tool.index}`); + if (!TOOL_NAME_PATTERN.test(name)) errors.push(`tool ${index} has unsafe name ${name}`); + if (names.has(name)) errors.push(`duplicate tool name: ${name}`); + names.add(name); + if (tool.definition.type !== "function") errors.push(`tool ${index} is not a function tool`); + if (tool.definition.function.parameters.additionalProperties !== false) { + errors.push(`tool ${index} parameters must reject additional properties`); + } + if (tool.handler.kind !== "deterministic_fixture_v1") { + errors.push(`tool ${index} has unsupported handler kind`); + } + }); + + const expectedHash = sha256Hex(canonicalJson(catalog.tools as unknown as JsonValue)); + if (catalog.tools_sha256 !== expectedHash) + errors.push("catalog tools_sha256 does not match tools"); + return errors; +} + +export function assertValidSyntheticCatalog(catalog: SyntheticCatalog): void { + const errors = validateSyntheticCatalog(catalog); + if (errors.length > 0) throw new Error(`invalid synthetic catalog: ${errors.join("; ")}`); +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-router.ts b/scripts/performance/tool-disclosure/compositional-tool-router.ts new file mode 100644 index 0000000000..4971b55f90 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-router.ts @@ -0,0 +1,698 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { performance } from "node:perf_hooks"; + +/** + * A small, independent compositional tool router for the performance-test + * harness. Model prompting and text embedding are injected so the routing + * mechanics remain provider- and hardware-neutral. + */ + +export const COMPOSITIONAL_ROUTER_TOP_K = 10; +export const COMPOSITIONAL_ROUTER_HINT_COUNT = 12; +export const COMPOSITIONAL_ROUTER_MAX_SELECTED_TOOLS = 16; +export const COMPOSITIONAL_ROUTER_MAX_SUBTASKS = 12; +export const COMPOSITIONAL_ROUTER_MAX_SUBTASK_CHARS = 512; + +export type DecompositionPass = "initial" | "refined"; + +export interface DecompositionRequest { + pass: DecompositionPass; + query: string; + /** Empty for the initial pass and populated only for the refinement pass. */ + tool_hints: readonly string[]; + signal?: AbortSignal; +} + +export interface TaskDecomposer { + /** Implementations return a JSON-like array; the router validates it at runtime. */ + decompose(request: DecompositionRequest): Promise; +} + +export interface TextEmbedder { + /** + * Return one dense vector per input string. The router normalizes every + * vector before exact inner-product search. + */ + embed(texts: readonly string[], signal?: AbortSignal): Promise; +} + +export interface ToolRoutingCatalogEntry { + name: string; + description: string; + /** Kept opaque and never copied into routing evidence. */ + definition: Definition; +} + +export interface DenseToolVector { + name: string; + vector: readonly number[]; +} + +export interface DenseToolIndex { + dimension: number; + tools: readonly DenseToolVector[]; +} + +export interface RankedTool { + name: string; + score: number; +} + +export type RoutingFallbackReason = + | "invalid-query" + | "invalid-catalog" + | "initial-decomposition-failed" + | "initial-decomposition-malformed" + | "catalog-embedding-failed" + | "subtask-embedding-failed" + | "no-candidates" + | "refinement-failed" + | "refinement-malformed" + | "refinement-no-tool-disagreement" + | "run-deadline-exceeded" + | "selection-limit-exceeded"; + +export interface CompositionalRoutingTimings { + initial_decomposition_ms: number; + catalog_embedding_ms: number; + initial_retrieval_ms: number; + refinement_ms: number; + final_retrieval_ms: number; + total_ms: number; +} + +/** Prompt-free evidence. Exact tool names require a reviewed public catalog. */ +export interface CompositionalRoutingEvidence { + initial_subtask_count: number; + refined_subtask_count: number; + decomposition_passes: number; + hint_count: number; + hint_tool_names: readonly string[]; + initial_candidate_counts: readonly number[]; + initial_candidate_tool_names: readonly (readonly string[])[]; + final_candidate_counts: readonly number[]; + final_candidate_tool_names: readonly (readonly string[])[]; + selected_tool_count: number; + selected_tool_names: readonly string[]; + fallback: RoutingFallbackReason | null; + timings: CompositionalRoutingTimings; +} + +export interface CompositionalRoutingResult { + /** `passthrough` tells the caller to preserve its complete unfiltered catalog. */ + disposition: "routed" | "passthrough"; + selected_tool_names: readonly string[]; + evidence: CompositionalRoutingEvidence; +} + +export interface CompositionalRouterOptions { + decomposer: TaskDecomposer; + embedder: TextEmbedder; + /** Zero runs the initial route; one adds a tool-informed refinement pass. */ + refinementPasses?: 0 | 1; + topK?: number; + hintCount?: number; + maxSelectedTools?: number; + /** Reuse a prebuilt normalized index instead of embedding the catalog per query. */ + preparedIndex?: DenseToolIndex | Promise; + signal?: AbortSignal; + clock?: () => number; +} + +export interface PairedCompositionalRoutingResult { + initial: CompositionalRoutingResult; + refined: CompositionalRoutingResult; + shared_initial_decomposition: true; + shared_initial_subtask_count: number; +} + +interface MutableEvidence { + initialSubtaskCount: number; + refinedSubtaskCount: number; + decompositionPasses: number; + hintNames: string[]; + initialCandidates: RankedTool[][]; + finalCandidates: RankedTool[][]; + selectedNames: string[]; + timings: CompositionalRoutingTimings; +} + +class RoutingFailure extends Error { + constructor(readonly reason: RoutingFallbackReason) { + super(reason); + } +} + +function compareNames(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + +function validatePositiveInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new TypeError(`${label} must be a positive safe integer`); + } +} + +function roundedMilliseconds(value: number): number { + return Number(Math.max(0, value).toFixed(3)); +} + +function elapsed(clock: () => number, startedAt: number): number { + return roundedMilliseconds(clock() - startedAt); +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new DOMException("routing aborted", "AbortError"); +} + +/** Normalize a finite, non-zero dense vector to unit L2 length. */ +export function l2Normalize(vector: readonly number[]): number[] { + if (!Array.isArray(vector) || vector.length === 0) { + throw new TypeError("embedding vectors must be non-empty arrays"); + } + let squaredNorm = 0; + for (const value of vector) { + if (!Number.isFinite(value)) throw new TypeError("embedding vectors must be finite"); + squaredNorm += value * value; + } + if (!Number.isFinite(squaredNorm) || squaredNorm <= 0) { + throw new TypeError("embedding vectors must have a positive finite L2 norm"); + } + const norm = Math.sqrt(squaredNorm); + return vector.map((value) => value / norm); +} + +function exactInnerProduct(left: readonly number[], right: readonly number[]): number { + if (left.length !== right.length) throw new TypeError("embedding dimensions do not match"); + let score = 0; + for (let index = 0; index < left.length; index += 1) score += left[index] * right[index]; + return Object.is(score, -0) ? 0 : score; +} + +function validateCatalog( + catalog: readonly ToolRoutingCatalogEntry[], +): void { + if (!Array.isArray(catalog)) throw new RoutingFailure("invalid-catalog"); + const names = new Set(); + for (const entry of catalog) { + if ( + !entry || + typeof entry.name !== "string" || + !entry.name.trim() || + entry.name !== entry.name.trim() || + typeof entry.description !== "string" || + names.has(entry.name) + ) { + throw new RoutingFailure("invalid-catalog"); + } + names.add(entry.name); + } +} + +function toolText(entry: ToolRoutingCatalogEntry): string { + return `${entry.name}\n${entry.description}`; +} + +function normalizedMatrix( + vectors: readonly (readonly number[])[], + expectedRows: number, +): { dimension: number; vectors: number[][] } { + if (!Array.isArray(vectors) || vectors.length !== expectedRows) { + throw new TypeError("embedder returned the wrong number of vectors"); + } + if (expectedRows === 0) return { dimension: 0, vectors: [] }; + const normalized = vectors.map((vector) => l2Normalize(vector)); + const dimension = normalized[0].length; + if (normalized.some((vector) => vector.length !== dimension)) { + throw new TypeError("embedder returned inconsistent vector dimensions"); + } + return { dimension, vectors: normalized }; +} + +/** Build a normalized dense index without retaining opaque tool definitions. */ +export async function buildDenseToolIndex( + catalog: readonly ToolRoutingCatalogEntry[], + embedder: TextEmbedder, + signal?: AbortSignal, +): Promise { + validateCatalog(catalog); + if (catalog.length === 0) return { dimension: 0, tools: [] }; + throwIfAborted(signal); + const embedded = await embedder.embed(catalog.map(toolText), signal); + throwIfAborted(signal); + const matrix = normalizedMatrix(embedded, catalog.length); + return { + dimension: matrix.dimension, + tools: catalog.map((entry, index) => ({ + name: entry.name, + vector: matrix.vectors[index], + })), + }; +} + +/** Rank a normalized exact inner-product index with deterministic name ties. */ +export function exactInnerProductTopK( + index: DenseToolIndex, + queryVector: readonly number[], + k = COMPOSITIONAL_ROUTER_TOP_K, +): RankedTool[] { + validatePositiveInteger(k, "k"); + if (index.tools.length === 0) return []; + const normalizedQuery = l2Normalize(queryVector); + if (normalizedQuery.length !== index.dimension) { + throw new TypeError("query and tool embedding dimensions do not match"); + } + return index.tools + .map((tool) => ({ + name: tool.name, + score: exactInnerProduct(normalizedQuery, tool.vector), + })) + .sort((left, right) => right.score - left.score || compareNames(left.name, right.name)) + .slice(0, k); +} + +/** Merge per-subtask candidates by maximum score, then apply a deterministic cap. */ +export function unionToolHints( + candidatesBySubtask: readonly (readonly RankedTool[])[], + limit = COMPOSITIONAL_ROUTER_HINT_COUNT, +): RankedTool[] { + validatePositiveInteger(limit, "hint limit"); + const maximumScores = new Map(); + for (const candidates of candidatesBySubtask) { + for (const candidate of candidates) { + if (!candidate.name || !Number.isFinite(candidate.score)) continue; + const current = maximumScores.get(candidate.name); + if (current === undefined || candidate.score > current) { + maximumScores.set(candidate.name, candidate.score); + } + } + } + return [...maximumScores.entries()] + .map(([name, score]) => ({ name, score })) + .sort((left, right) => right.score - left.score || compareNames(left.name, right.name)) + .slice(0, limit); +} + +function validateSubtasks(value: unknown, malformedReason: RoutingFallbackReason): string[] { + if (!Array.isArray(value) || value.length > COMPOSITIONAL_ROUTER_MAX_SUBTASKS) { + throw new RoutingFailure(malformedReason); + } + const subtasks: string[] = []; + for (const item of value) { + if (typeof item !== "string") throw new RoutingFailure(malformedReason); + const normalized = item.trim(); + if (!normalized || normalized.length > COMPOSITIONAL_ROUTER_MAX_SUBTASK_CHARS) { + throw new RoutingFailure(malformedReason); + } + subtasks.push(normalized); + } + return subtasks; +} + +async function embedSubtasks( + subtasks: readonly string[], + embedder: TextEmbedder, + index: DenseToolIndex, + signal?: AbortSignal, +): Promise { + throwIfAborted(signal); + const embedded = await embedder.embed(subtasks, signal); + throwIfAborted(signal); + const matrix = normalizedMatrix(embedded, subtasks.length); + if (matrix.dimension !== index.dimension) { + throw new TypeError("subtask and catalog embedding dimensions do not match"); + } + return matrix.vectors; +} + +function emptyTimings(): CompositionalRoutingTimings { + return { + initial_decomposition_ms: 0, + catalog_embedding_ms: 0, + initial_retrieval_ms: 0, + refinement_ms: 0, + final_retrieval_ms: 0, + total_ms: 0, + }; +} + +function evidence( + mutable: MutableEvidence, + fallback: RoutingFallbackReason | null, +): CompositionalRoutingEvidence { + return { + initial_subtask_count: mutable.initialSubtaskCount, + refined_subtask_count: mutable.refinedSubtaskCount, + decomposition_passes: mutable.decompositionPasses, + hint_count: mutable.hintNames.length, + hint_tool_names: [...mutable.hintNames], + initial_candidate_counts: mutable.initialCandidates.map((candidates) => candidates.length), + initial_candidate_tool_names: mutable.initialCandidates.map((candidates) => + candidates.map((candidate) => candidate.name), + ), + final_candidate_counts: mutable.finalCandidates.map((candidates) => candidates.length), + final_candidate_tool_names: mutable.finalCandidates.map((candidates) => + candidates.map((candidate) => candidate.name), + ), + selected_tool_count: mutable.selectedNames.length, + selected_tool_names: [...mutable.selectedNames], + fallback, + timings: { ...mutable.timings }, + }; +} + +function uniqueInOrder(values: readonly string[]): string[] { + const seen = new Set(); + return values.filter((value) => { + if (seen.has(value)) return false; + seen.add(value); + return true; + }); +} + +interface RoutingLimits { + topK: number; + hintCount: number; + maxSelectedTools: number; +} + +interface InitialRoutingStage { + initialSubtasks: readonly string[]; + index: DenseToolIndex | null; + mutable: MutableEvidence; + fallback: RoutingFallbackReason | null; +} + +function routingLimits(options: CompositionalRouterOptions): RoutingLimits { + const limits = { + topK: options.topK ?? COMPOSITIONAL_ROUTER_TOP_K, + hintCount: options.hintCount ?? COMPOSITIONAL_ROUTER_HINT_COUNT, + maxSelectedTools: options.maxSelectedTools ?? COMPOSITIONAL_ROUTER_MAX_SELECTED_TOOLS, + }; + validatePositiveInteger(limits.topK, "topK"); + validatePositiveInteger(limits.hintCount, "hintCount"); + validatePositiveInteger(limits.maxSelectedTools, "maxSelectedTools"); + return limits; +} + +function emptyMutableEvidence(): MutableEvidence { + return { + initialSubtaskCount: 0, + refinedSubtaskCount: 0, + decompositionPasses: 0, + hintNames: [], + initialCandidates: [], + finalCandidates: [], + selectedNames: [], + timings: emptyTimings(), + }; +} + +function cloneCandidates(candidates: readonly (readonly RankedTool[])[]): RankedTool[][] { + return candidates.map((ranked) => ranked.map((candidate) => ({ ...candidate }))); +} + +function cloneMutableEvidence(source: MutableEvidence): MutableEvidence { + return { + initialSubtaskCount: source.initialSubtaskCount, + refinedSubtaskCount: source.refinedSubtaskCount, + decompositionPasses: source.decompositionPasses, + hintNames: [...source.hintNames], + initialCandidates: cloneCandidates(source.initialCandidates), + finalCandidates: cloneCandidates(source.finalCandidates), + selectedNames: [...source.selectedNames], + timings: { ...source.timings }, + }; +} + +function fallbackReason(error: unknown): RoutingFallbackReason { + return error instanceof RoutingFailure ? error.reason : "subtask-embedding-failed"; +} + +async function prepareInitialRoutingStage( + query: string, + catalog: readonly ToolRoutingCatalogEntry[], + options: Omit, + limits: RoutingLimits, + clock: () => number, +): Promise { + const mutable = emptyMutableEvidence(); + let initialSubtasks: string[] = []; + let index: DenseToolIndex | null = null; + try { + throwIfAborted(options.signal); + if (typeof query !== "string" || !query.trim()) throw new RoutingFailure("invalid-query"); + validateCatalog(catalog); + + const initialStartedAt = clock(); + let rawInitial: unknown; + mutable.decompositionPasses = 1; + try { + rawInitial = await options.decomposer.decompose({ + pass: "initial", + query, + tool_hints: [], + ...(options.signal ? { signal: options.signal } : {}), + }); + } catch { + throw new RoutingFailure("initial-decomposition-failed"); + } finally { + mutable.timings.initial_decomposition_ms = elapsed(clock, initialStartedAt); + } + initialSubtasks = validateSubtasks(rawInitial, "initial-decomposition-malformed"); + mutable.initialSubtaskCount = initialSubtasks.length; + if (initialSubtasks.length === 0) { + return { initialSubtasks, index, mutable, fallback: null }; + } + + const catalogStartedAt = clock(); + try { + index = await (options.preparedIndex ?? + buildDenseToolIndex(catalog, options.embedder, options.signal)); + const catalogNames = catalog.map((entry) => entry.name); + if ( + index.tools.length !== catalogNames.length || + index.tools.some((tool, position) => tool.name !== catalogNames[position]) + ) { + throw new TypeError("prepared index does not match the routing catalog"); + } + } catch { + throw new RoutingFailure("catalog-embedding-failed"); + } finally { + mutable.timings.catalog_embedding_ms = elapsed(clock, catalogStartedAt); + } + if (index.tools.length === 0) throw new RoutingFailure("no-candidates"); + + const initialRetrievalStartedAt = clock(); + try { + const vectors = await embedSubtasks(initialSubtasks, options.embedder, index, options.signal); + mutable.initialCandidates = vectors.map((vector) => + exactInnerProductTopK( + index as DenseToolIndex, + vector, + Math.max(limits.hintCount, limits.topK), + ), + ); + } catch { + throw new RoutingFailure("subtask-embedding-failed"); + } finally { + mutable.timings.initial_retrieval_ms = elapsed(clock, initialRetrievalStartedAt); + } + return { initialSubtasks, index, mutable, fallback: null }; + } catch (error) { + return { initialSubtasks, index, mutable, fallback: fallbackReason(error) }; + } +} + +async function routeFromInitialStage( + query: string, + catalog: readonly ToolRoutingCatalogEntry[], + options: Omit, + refinementPasses: 0 | 1, + limits: RoutingLimits, + clock: () => number, + totalStartedAt: number, + stage: InitialRoutingStage, +): Promise { + const mutable = cloneMutableEvidence(stage.mutable); + const finish = ( + disposition: CompositionalRoutingResult["disposition"], + fallback: RoutingFallbackReason | null, + ): CompositionalRoutingResult => { + mutable.timings.total_ms = elapsed(clock, totalStartedAt); + return { + disposition, + selected_tool_names: disposition === "routed" ? [...mutable.selectedNames] : [], + evidence: evidence(mutable, fallback), + }; + }; + + if (stage.fallback !== null) return finish("passthrough", stage.fallback); + if (stage.initialSubtasks.length === 0) return finish("routed", null); + const index = stage.index; + if (index === null) return finish("passthrough", "no-candidates"); + + try { + if (refinementPasses === 0) { + mutable.finalCandidates = mutable.initialCandidates.map((candidates) => + candidates.slice(0, limits.topK), + ); + const selectedNames = uniqueInOrder( + mutable.finalCandidates.map((candidates) => candidates[0]?.name).filter(Boolean), + ); + if (selectedNames.length > limits.maxSelectedTools) { + throw new RoutingFailure("selection-limit-exceeded"); + } + mutable.selectedNames = selectedNames; + return finish("routed", null); + } + + const hints = unionToolHints(mutable.initialCandidates, limits.hintCount); + mutable.hintNames = hints.map((candidate) => candidate.name); + if (mutable.hintNames.length === 0) throw new RoutingFailure("no-candidates"); + + const refinementStartedAt = clock(); + let rawRefined: unknown; + mutable.decompositionPasses = 2; + try { + rawRefined = await options.decomposer.decompose({ + pass: "refined", + query, + tool_hints: mutable.hintNames, + ...(options.signal ? { signal: options.signal } : {}), + }); + } catch { + throw new RoutingFailure("refinement-failed"); + } finally { + mutable.timings.refinement_ms = elapsed(clock, refinementStartedAt); + } + const refinedSubtasks = validateSubtasks(rawRefined, "refinement-malformed"); + mutable.refinedSubtaskCount = refinedSubtasks.length; + if (refinedSubtasks.length === 0) { + throw new RoutingFailure("refinement-no-tool-disagreement"); + } + + const finalRetrievalStartedAt = clock(); + try { + const vectors = await embedSubtasks(refinedSubtasks, options.embedder, index, options.signal); + mutable.finalCandidates = vectors.map((vector) => + exactInnerProductTopK(index, vector, limits.topK), + ); + } catch { + throw new RoutingFailure("subtask-embedding-failed"); + } finally { + mutable.timings.final_retrieval_ms = elapsed(clock, finalRetrievalStartedAt); + } + if (mutable.finalCandidates.some((candidates) => candidates.length === 0)) { + throw new RoutingFailure("no-candidates"); + } + + const selectedNames = uniqueInOrder( + mutable.finalCandidates.map((candidates) => candidates[0]?.name).filter(Boolean), + ); + if (selectedNames.length > limits.maxSelectedTools) { + throw new RoutingFailure("selection-limit-exceeded"); + } + mutable.selectedNames = selectedNames; + return finish("routed", null); + } catch (error) { + return finish("passthrough", fallbackReason(error)); + } +} + +/** + * Run an initial decomposition and, by default, one tool-informed refinement. + * Set `refinementPasses` to zero to score the initial route. Any malformed model + * output or unavailable retrieval dependency returns `passthrough`, so a caller + * can retain the complete original tool catalog rather than losing a capability. + * An empty initial decomposition is treated as an intentional no-tool route. + */ +export async function routeCompositionalTools( + query: string, + catalog: readonly ToolRoutingCatalogEntry[], + options: CompositionalRouterOptions, +): Promise { + const limits = routingLimits(options); + const refinementPasses = options.refinementPasses ?? 1; + if (refinementPasses !== 0 && refinementPasses !== 1) { + throw new TypeError("refinementPasses must be zero or one"); + } + const clock = options.clock ?? (() => performance.now()); + const totalStartedAt = clock(); + const sharedOptions = { ...options, refinementPasses: undefined }; + const stage = await prepareInitialRoutingStage(query, catalog, sharedOptions, limits, clock); + return routeFromInitialStage( + query, + catalog, + sharedOptions, + refinementPasses, + limits, + clock, + totalStartedAt, + stage, + ); +} + +/** + * Produce a fair initial/refined pair from one initial decomposition and one + * shared catalog index and initial retrieval stage. + */ +export async function routeCompositionalToolsPaired( + query: string, + catalog: readonly ToolRoutingCatalogEntry[], + options: Omit, +): Promise { + const limits = routingLimits(options); + const clock = options.clock ?? (() => performance.now()); + const totalStartedAt = clock(); + const stage = await prepareInitialRoutingStage(query, catalog, options, limits, clock); + const initial = await routeFromInitialStage( + query, + catalog, + options, + 0, + limits, + clock, + totalStartedAt, + stage, + ); + const refined = await routeFromInitialStage( + query, + catalog, + options, + 1, + limits, + clock, + totalStartedAt, + stage, + ); + if (initial.evidence.initial_subtask_count !== refined.evidence.initial_subtask_count) { + throw new Error("paired routes did not share the initial decomposition"); + } + return { + initial, + refined, + shared_initial_decomposition: true, + shared_initial_subtask_count: initial.evidence.initial_subtask_count, + }; +} + +/** + * Resolve selected names back to the caller-owned definitions. Routed results + * preserve selected order; passthrough results preserve the full catalog order. + */ +export function selectRoutedToolDefinitions( + catalog: readonly ToolRoutingCatalogEntry[], + result: CompositionalRoutingResult, +): Definition[] { + if (result.disposition === "passthrough") return catalog.map((entry) => entry.definition); + const byName = new Map(catalog.map((entry) => [entry.name, entry.definition])); + return result.selected_tool_names.flatMap((name) => { + const definition = byName.get(name); + return definition === undefined ? [] : [definition]; + }); +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-acceptance.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-acceptance.ts new file mode 100644 index 0000000000..4ef96345b6 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-acceptance.ts @@ -0,0 +1,649 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { OpenAIFunctionTool } from "./catalog"; +import type { RoutingFallbackReason } from "./compositional-tool-router"; + +export type CompositionalRoutingVariant = "initial" | "refined"; + +export interface CompositionalRoutingAcceptanceTool { + name: string; + description: string; + definition: OpenAIFunctionTool; +} + +export interface CompositionalRoutingExpectedStep { + capability: string; + /** A lower-case phrase that makes this capability observable in the prompt. */ + prompt_cue: string; + tool_name: string; +} + +export interface CompositionalRoutingAcceptanceCase { + id: string; + prompt: string; + expected_steps: readonly CompositionalRoutingExpectedStep[]; +} + +export interface CompositionalRoutingStepPrediction { + subtask: string; + /** Best candidate first. The evaluator reads positions 1 and 1-10. */ + ranked_tool_names: readonly string[]; +} + +export interface CompositionalRoutingCasePrediction { + case_id: string; + disposition: "routed" | "passthrough"; + fallback: RoutingFallbackReason | null; + steps: readonly CompositionalRoutingStepPrediction[]; + /** Logical router selection; passthrough forwarding is derived from the catalog. */ + selected_tool_names: readonly string[]; +} + +export interface CompositionalRoutingVariantInput { + variant: CompositionalRoutingVariant; + cases: readonly CompositionalRoutingCasePrediction[]; +} + +export interface CompositionalRoutingCaseResult { + case_id: string; + disposition: "routed" | "passthrough"; + fallback: RoutingFallbackReason | null; + routing_succeeded: boolean; + expected_step_count: number; + predicted_step_count: number; + decomposition_exact: boolean; + decomposition_within_one: boolean; + /** One-based rank for each expected step; null means the target was absent. */ + expected_tool_ranks: readonly (number | null)[]; + chain_exact_selection: boolean; + selected_tool_count: number; + selection_count_exact: boolean; + forwarded_tool_names: readonly string[]; + forwarded_tool_count: number; + chain_exact_forwarding: boolean; + forwarding_count_exact: boolean; + /** Null for cases that require tools. */ + no_tool_exact: boolean | null; +} + +export interface CompositionalRoutingMetrics { + variant: CompositionalRoutingVariant; + case_count: number; + route_failure_case_count: number; + routing_success_rate: number; + expected_step_count: number; + predicted_step_count: number; + decomposition_exact_rate: number; + decomposition_within_one_rate: number; + exact_tool_recall_at_1: number; + exact_tool_recall_at_10: number; + chain_exact_selection_rate: number; + selection_count_exact_rate: number; + chain_exact_forwarding_rate: number; + forwarding_count_exact_rate: number; + selected_tool_count: number; + mean_selected_tool_count: number; + max_selected_tool_count: number; + forwarded_tool_count: number; + mean_forwarded_tool_count: number; + max_forwarded_tool_count: number; + no_tool_case_count: number; + no_tool_exact_rate: number; + cases: readonly CompositionalRoutingCaseResult[]; +} + +export interface CompositionalRoutingGateThresholds { + min_routing_success_rate: number; + min_decomposition_exact_rate: number; + min_decomposition_within_one_rate: number; + min_exact_tool_recall_at_1: number; + min_exact_tool_recall_at_10: number; + min_chain_exact_selection_rate: number; + min_selection_count_exact_rate: number; + min_chain_exact_forwarding_rate: number; + min_forwarding_count_exact_rate: number; + min_no_tool_exact_rate: number; + max_selected_tool_count: number; + max_forwarded_tool_count: number; +} + +export interface CompositionalRoutingGateResult { + passed: boolean; + thresholds: CompositionalRoutingGateThresholds; + reasons: readonly string[]; +} + +export interface CompositionalRoutingComparison { + initial: CompositionalRoutingMetrics; + refined: CompositionalRoutingMetrics; + refined_minus_initial: { + decomposition_exact_rate: number; + decomposition_within_one_rate: number; + exact_tool_recall_at_1: number; + exact_tool_recall_at_10: number; + chain_exact_selection_rate: number; + chain_exact_forwarding_rate: number; + routing_success_rate: number; + }; + refined_not_worse: boolean; + refined_gate: CompositionalRoutingGateResult; + passed: boolean; + reasons: readonly string[]; +} + +export const DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS: CompositionalRoutingGateThresholds = { + min_routing_success_rate: 1, + min_decomposition_exact_rate: 1, + min_decomposition_within_one_rate: 1, + min_exact_tool_recall_at_1: 1, + min_exact_tool_recall_at_10: 1, + min_chain_exact_selection_rate: 1, + min_selection_count_exact_rate: 1, + min_chain_exact_forwarding_rate: 1, + min_forwarding_count_exact_rate: 1, + min_no_tool_exact_rate: 1, + max_selected_tool_count: 5, + max_forwarded_tool_count: 5, +}; + +function routeTool(name: string, description: string): CompositionalRoutingAcceptanceTool { + return { + name, + description, + definition: { + type: "function", + function: { + name, + description, + parameters: { + type: "object", + description: "Route-only acceptance fixture input.", + properties: { + input: { + type: "string", + description: "Deterministic input for the selected fixture capability.", + minLength: 1, + maxLength: 160, + }, + }, + required: ["input"], + additionalProperties: false, + }, + }, + }, + }; +} + +/** + * A small route-only catalog with deliberately distinct capability metadata. + * It complements the scaling catalog, whose repeated category/operation pairs + * are useful for schema-volume tests but cannot identify one exact semantic + * target without relying on numeric suffixes. + */ +export const COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS: readonly CompositionalRoutingAcceptanceTool[] = + [ + routeTool( + "route_fetch_https_bytes", + "Download raw response bytes from one HTTPS address without interpreting the payload.", + ), + routeTool( + "route_read_local_file", + "Read bytes from a file already present on the local filesystem.", + ), + routeTool( + "route_decompress_gzip", + "Expand a gzip-compressed byte stream into its original uncompressed bytes.", + ), + routeTool("route_decompress_zip", "Extract named members from a ZIP archive container."), + routeTool( + "route_parse_csv_rows", + "Parse comma-separated text with a header row into structured records.", + ), + routeTool( + "route_parse_jsonl_records", + "Parse newline-delimited JSON text into one structured record per line.", + ), + routeTool( + "route_validate_json_schema", + "Check structured records against a supplied JSON Schema contract.", + ), + routeTool( + "route_filter_tabular_rows", + "Keep only table rows that satisfy a field-level predicate.", + ), + routeTool( + "route_aggregate_numeric_column", + "Compute a requested numeric aggregate such as an average over one table column.", + ), + routeTool("route_render_line_chart", "Render ordered numeric observations as a line chart."), + routeTool("route_render_bar_chart", "Render categorical numeric values as vertical bars."), + routeTool( + "route_lookup_directory_contact", + "Find a person's messaging destination in an organization directory.", + ), + routeTool( + "route_draft_email_message", + "Create an email draft with recipients, subject, and body without sending it.", + ), + routeTool("route_send_email_message", "Deliver an existing email draft to its recipients."), + routeTool("route_post_team_channel", "Post a text update to a team chat channel destination."), + routeTool("route_store_object_blob", "Write supplied bytes to an object-storage key."), + routeTool( + "route_query_sql_readonly", + "Run a read-only SQL query and return tabular result rows.", + ), + routeTool("route_summarize_text", "Condense a text document into a short prose summary."), + routeTool("route_transcribe_audio", "Convert spoken audio into a text transcript."), + routeTool("route_translate_text", "Translate supplied text from one language to another."), + ]; + +function step( + toolName: string, + capability: string, + promptCue: string, +): CompositionalRoutingExpectedStep { + return { tool_name: toolName, capability, prompt_cue: promptCue }; +} + +/** Eight unambiguous cases spanning zero through five implicit atomic capabilities. */ +export const COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES: readonly CompositionalRoutingAcceptanceCase[] = + [ + { + id: "route-no-tool-01", + prompt: "Use no external capability and reply with the literal phrase ROUTE-CONTROL-ALPHA.", + expected_steps: [], + }, + { + id: "route-single-csv-01", + prompt: "Turn comma-separated text with a header row into structured records.", + expected_steps: [step("route_parse_csv_rows", "parse CSV records", "comma-separated")], + }, + { + id: "route-single-bars-01", + prompt: "Show the categorical totals as vertical bars, rather than a connected line.", + expected_steps: [ + step( + "route_render_bar_chart", + "render categorical values as vertical bars", + "vertical bars", + ), + ], + }, + { + id: "route-chain-two-channel-01", + prompt: + "Find Morgan's team-chat destination in the organization directory, then post the status update to that channel.", + expected_steps: [ + step("route_lookup_directory_contact", "look up a directory contact", "directory"), + step("route_post_team_channel", "post a team-channel message", "post"), + ], + }, + { + id: "route-chain-two-email-01", + prompt: "Prepare an email draft for the reviewers, then deliver that draft to them.", + expected_steps: [ + step("route_draft_email_message", "draft an email", "draft"), + step("route_send_email_message", "send an email draft", "deliver"), + ], + }, + { + id: "route-chain-three-analysis-01", + prompt: + "Run a read-only SQL query for monthly latency, calculate the average value, and plot the ordered results as a line chart.", + expected_steps: [ + step("route_query_sql_readonly", "query SQL data", "sql"), + step("route_aggregate_numeric_column", "aggregate a numeric column", "average"), + step("route_render_line_chart", "render a line chart", "line chart"), + ], + }, + { + id: "route-chain-four-jsonl-01", + prompt: + "Download the bytes at the HTTPS address, expand the gzip stream, parse its newline-delimited JSON records, and check them against the supplied JSON Schema.", + expected_steps: [ + step("route_fetch_https_bytes", "fetch HTTPS bytes", "https"), + step("route_decompress_gzip", "decompress gzip bytes", "gzip"), + step("route_parse_jsonl_records", "parse JSONL records", "newline-delimited json"), + step("route_validate_json_schema", "validate JSON records", "json schema"), + ], + }, + { + id: "route-chain-five-report-01", + prompt: + "Fetch the CSV file from its HTTPS address, parse the comma-separated rows, keep rows whose state is active, average the cost column, and present the result as vertical bars.", + expected_steps: [ + step("route_fetch_https_bytes", "fetch HTTPS bytes", "https"), + step("route_parse_csv_rows", "parse CSV records", "comma-separated"), + step("route_filter_tabular_rows", "filter table rows", "keep rows"), + step("route_aggregate_numeric_column", "aggregate a numeric column", "average"), + step( + "route_render_bar_chart", + "render categorical values as vertical bars", + "vertical bars", + ), + ], + }, + ]; + +function assertRate(value: number, label: string): void { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new RangeError(`${label} must be between zero and one`); + } +} + +function assertThresholds(thresholds: CompositionalRoutingGateThresholds): void { + for (const [label, value] of Object.entries(thresholds)) { + if (label === "max_selected_tool_count" || label === "max_forwarded_tool_count") continue; + assertRate(value, label); + } + for (const [label, value] of [ + ["max_selected_tool_count", thresholds.max_selected_tool_count], + ["max_forwarded_tool_count", thresholds.max_forwarded_tool_count], + ] as const) { + if (!Number.isSafeInteger(value)) throw new TypeError(`${label} must be a safe integer`); + if (value < 0) throw new RangeError(`${label} must not be negative`); + } +} + +function assertUnique(values: readonly string[], label: string): void { + if (new Set(values).size !== values.length) throw new Error(`${label} contains duplicates`); +} + +function assertFixture( + tools: readonly CompositionalRoutingAcceptanceTool[], + cases: readonly CompositionalRoutingAcceptanceCase[], +): Set { + if (tools.length === 0) throw new Error("route acceptance catalog must not be empty"); + if (cases.length === 0) throw new Error("route acceptance cases must not be empty"); + const toolNames = tools.map((tool) => tool.name); + assertUnique(toolNames, "route acceptance tool names"); + const knownTools = new Set(toolNames); + const caseIds = cases.map((fixture) => fixture.id); + assertUnique(caseIds, "route acceptance case ids"); + for (const fixture of cases) { + if (!fixture.prompt.trim()) throw new Error(`${fixture.id} has an empty prompt`); + const expectedNames = fixture.expected_steps.map((expected) => expected.tool_name); + assertUnique(expectedNames, `${fixture.id} expected tool names`); + for (const expected of fixture.expected_steps) { + if (!knownTools.has(expected.tool_name)) { + throw new Error(`${fixture.id} references unknown tool ${expected.tool_name}`); + } + if (!expected.capability.trim() || !expected.prompt_cue.trim()) { + throw new Error(`${fixture.id} has empty expected step metadata`); + } + if (!fixture.prompt.toLocaleLowerCase("en-US").includes(expected.prompt_cue)) { + throw new Error(`${fixture.id} does not expose prompt cue ${expected.prompt_cue}`); + } + } + } + return knownTools; +} + +function assertPrediction( + prediction: CompositionalRoutingCasePrediction, + knownTools: ReadonlySet, +): void { + if (prediction.disposition !== "routed" && prediction.disposition !== "passthrough") { + throw new TypeError(`${prediction.case_id} has an invalid disposition`); + } + const routingSucceeded = prediction.disposition === "routed"; + if (routingSucceeded !== (prediction.fallback === null)) { + throw new Error(`${prediction.case_id} has inconsistent disposition and fallback`); + } + assertUnique(prediction.selected_tool_names, `${prediction.case_id} selected tools`); + for (const name of prediction.selected_tool_names) { + if (!knownTools.has(name)) { + throw new Error(`${prediction.case_id} selected unknown tool ${name}`); + } + } + for (const [index, predicted] of prediction.steps.entries()) { + if (!predicted.subtask.trim()) { + throw new Error(`${prediction.case_id} step ${index + 1} has an empty subtask`); + } + assertUnique(predicted.ranked_tool_names, `${prediction.case_id} step ${index + 1} ranking`); + for (const name of predicted.ranked_tool_names) { + if (!knownTools.has(name)) { + throw new Error(`${prediction.case_id} ranked unknown tool ${name}`); + } + } + } +} + +function rate(numerator: number, denominator: number): number { + return denominator === 0 ? 1 : numerator / denominator; +} + +function sequencesEqual(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +/** Evaluate one router variant without invoking a model or embedding implementation. */ +export function evaluateCompositionalRoutingVariant( + input: CompositionalRoutingVariantInput, + options: { + tools?: readonly CompositionalRoutingAcceptanceTool[]; + cases?: readonly CompositionalRoutingAcceptanceCase[]; + } = {}, +): CompositionalRoutingMetrics { + const tools = options.tools ?? COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS; + const fixtures = options.cases ?? COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES; + const knownTools = assertFixture(tools, fixtures); + const catalogToolNames = tools.map((tool) => tool.name); + if (input.cases.length !== fixtures.length) { + throw new Error(`${input.variant} predictions must contain exactly ${fixtures.length} cases`); + } + const predictions = new Map(); + for (const prediction of input.cases) { + if (predictions.has(prediction.case_id)) { + throw new Error(`${input.variant} contains duplicate case ${prediction.case_id}`); + } + assertPrediction(prediction, knownTools); + predictions.set(prediction.case_id, prediction); + } + + const results: CompositionalRoutingCaseResult[] = []; + let expectedStepCount = 0; + let predictedStepCount = 0; + let routingSuccesses = 0; + let routeFailureCases = 0; + let exactDecomposition = 0; + let withinOneDecomposition = 0; + let recallAtOne = 0; + let recallAtTen = 0; + let exactChains = 0; + let exactSelectionCounts = 0; + let exactForwardingChains = 0; + let exactForwardingCounts = 0; + let selectedToolCount = 0; + let maxSelectedToolCount = 0; + let forwardedToolCount = 0; + let maxForwardedToolCount = 0; + let noToolCaseCount = 0; + let exactNoToolCases = 0; + + for (const fixture of fixtures) { + const prediction = predictions.get(fixture.id); + if (!prediction) throw new Error(`${input.variant} is missing case ${fixture.id}`); + const expectedNames = fixture.expected_steps.map((expected) => expected.tool_name); + const expectedCount = expectedNames.length; + const predictedCount = prediction.steps.length; + const routingSucceeded = prediction.disposition === "routed"; + const forwardedToolNames = routingSucceeded + ? [...prediction.selected_tool_names] + : [...catalogToolNames]; + const decompositionExact = predictedCount === expectedCount; + const decompositionWithinOne = Math.abs(predictedCount - expectedCount) <= 1; + const expectedToolRanks = expectedNames.map((expectedName, index) => { + const ranked = prediction.steps[index]?.ranked_tool_names ?? []; + const rank = ranked.indexOf(expectedName); + if (rank === 0) recallAtOne += 1; + if (rank >= 0 && rank < 10) recallAtTen += 1; + return rank < 0 ? null : rank + 1; + }); + const chainExactSelection = sequencesEqual(prediction.selected_tool_names, expectedNames); + const selectionCountExact = prediction.selected_tool_names.length === expectedCount; + const chainExactForwarding = + routingSucceeded && sequencesEqual(forwardedToolNames, expectedNames); + const forwardingCountExact = routingSucceeded && forwardedToolNames.length === expectedCount; + const noToolExact = + expectedCount === 0 + ? routingSucceeded && predictedCount === 0 && forwardedToolNames.length === 0 + : null; + + expectedStepCount += expectedCount; + predictedStepCount += predictedCount; + routingSuccesses += Number(routingSucceeded); + routeFailureCases += Number(!routingSucceeded); + exactDecomposition += Number(decompositionExact); + withinOneDecomposition += Number(decompositionWithinOne); + exactChains += Number(chainExactSelection); + exactSelectionCounts += Number(selectionCountExact); + exactForwardingChains += Number(chainExactForwarding); + exactForwardingCounts += Number(forwardingCountExact); + selectedToolCount += prediction.selected_tool_names.length; + maxSelectedToolCount = Math.max(maxSelectedToolCount, prediction.selected_tool_names.length); + forwardedToolCount += forwardedToolNames.length; + maxForwardedToolCount = Math.max(maxForwardedToolCount, forwardedToolNames.length); + if (noToolExact !== null) { + noToolCaseCount += 1; + exactNoToolCases += Number(noToolExact); + } + results.push({ + case_id: fixture.id, + disposition: prediction.disposition, + fallback: prediction.fallback, + routing_succeeded: routingSucceeded, + expected_step_count: expectedCount, + predicted_step_count: predictedCount, + decomposition_exact: decompositionExact, + decomposition_within_one: decompositionWithinOne, + expected_tool_ranks: expectedToolRanks, + chain_exact_selection: chainExactSelection, + selected_tool_count: prediction.selected_tool_names.length, + selection_count_exact: selectionCountExact, + forwarded_tool_names: forwardedToolNames, + forwarded_tool_count: forwardedToolNames.length, + chain_exact_forwarding: chainExactForwarding, + forwarding_count_exact: forwardingCountExact, + no_tool_exact: noToolExact, + }); + } + + return { + variant: input.variant, + case_count: fixtures.length, + route_failure_case_count: routeFailureCases, + routing_success_rate: rate(routingSuccesses, fixtures.length), + expected_step_count: expectedStepCount, + predicted_step_count: predictedStepCount, + decomposition_exact_rate: rate(exactDecomposition, fixtures.length), + decomposition_within_one_rate: rate(withinOneDecomposition, fixtures.length), + exact_tool_recall_at_1: rate(recallAtOne, expectedStepCount), + exact_tool_recall_at_10: rate(recallAtTen, expectedStepCount), + chain_exact_selection_rate: rate(exactChains, fixtures.length), + selection_count_exact_rate: rate(exactSelectionCounts, fixtures.length), + chain_exact_forwarding_rate: rate(exactForwardingChains, fixtures.length), + forwarding_count_exact_rate: rate(exactForwardingCounts, fixtures.length), + selected_tool_count: selectedToolCount, + mean_selected_tool_count: selectedToolCount / fixtures.length, + max_selected_tool_count: maxSelectedToolCount, + forwarded_tool_count: forwardedToolCount, + mean_forwarded_tool_count: forwardedToolCount / fixtures.length, + max_forwarded_tool_count: maxForwardedToolCount, + no_tool_case_count: noToolCaseCount, + no_tool_exact_rate: rate(exactNoToolCases, noToolCaseCount), + cases: results, + }; +} + +/** Apply explicit, deterministic quality and disclosure-size gates. */ +export function evaluateCompositionalRoutingGates( + metrics: CompositionalRoutingMetrics, + thresholds: CompositionalRoutingGateThresholds = DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS, +): CompositionalRoutingGateResult { + assertThresholds(thresholds); + const reasons: string[] = []; + if (metrics.route_failure_case_count > 0) { + reasons.push(`route_failure_case_count ${metrics.route_failure_case_count} must be zero`); + } + const minimums: ReadonlyArray< + [keyof CompositionalRoutingMetrics, keyof CompositionalRoutingGateThresholds] + > = [ + ["routing_success_rate", "min_routing_success_rate"], + ["decomposition_exact_rate", "min_decomposition_exact_rate"], + ["decomposition_within_one_rate", "min_decomposition_within_one_rate"], + ["exact_tool_recall_at_1", "min_exact_tool_recall_at_1"], + ["exact_tool_recall_at_10", "min_exact_tool_recall_at_10"], + ["chain_exact_selection_rate", "min_chain_exact_selection_rate"], + ["selection_count_exact_rate", "min_selection_count_exact_rate"], + ["chain_exact_forwarding_rate", "min_chain_exact_forwarding_rate"], + ["forwarding_count_exact_rate", "min_forwarding_count_exact_rate"], + ["no_tool_exact_rate", "min_no_tool_exact_rate"], + ]; + for (const [metricName, thresholdName] of minimums) { + const value = metrics[metricName]; + const threshold = thresholds[thresholdName]; + if (typeof value !== "number" || value < threshold) { + reasons.push(`${String(metricName)} ${String(value)} is below ${String(threshold)}`); + } + } + if (metrics.max_selected_tool_count > thresholds.max_selected_tool_count) { + reasons.push( + `max_selected_tool_count ${metrics.max_selected_tool_count} exceeds ${thresholds.max_selected_tool_count}`, + ); + } + if (metrics.max_forwarded_tool_count > thresholds.max_forwarded_tool_count) { + reasons.push( + `max_forwarded_tool_count ${metrics.max_forwarded_tool_count} exceeds ${thresholds.max_forwarded_tool_count}`, + ); + } + return { passed: reasons.length === 0, thresholds: { ...thresholds }, reasons }; +} + +/** Compare initial and one-pass refined routing while gating refined independently. */ +export function compareCompositionalRoutingVariants( + initialInput: CompositionalRoutingVariantInput, + refinedInput: CompositionalRoutingVariantInput, + options: { + tools?: readonly CompositionalRoutingAcceptanceTool[]; + cases?: readonly CompositionalRoutingAcceptanceCase[]; + thresholds?: CompositionalRoutingGateThresholds; + } = {}, +): CompositionalRoutingComparison { + if (initialInput.variant !== "initial") { + throw new Error("the first route comparison input must be initial"); + } + if (refinedInput.variant !== "refined") { + throw new Error("the second route comparison input must be refined"); + } + const evaluateOptions = { tools: options.tools, cases: options.cases }; + const initial = evaluateCompositionalRoutingVariant(initialInput, evaluateOptions); + const refined = evaluateCompositionalRoutingVariant(refinedInput, evaluateOptions); + const refinedGate = evaluateCompositionalRoutingGates(refined, options.thresholds); + const deltas = { + decomposition_exact_rate: refined.decomposition_exact_rate - initial.decomposition_exact_rate, + decomposition_within_one_rate: + refined.decomposition_within_one_rate - initial.decomposition_within_one_rate, + exact_tool_recall_at_1: refined.exact_tool_recall_at_1 - initial.exact_tool_recall_at_1, + exact_tool_recall_at_10: refined.exact_tool_recall_at_10 - initial.exact_tool_recall_at_10, + chain_exact_selection_rate: + refined.chain_exact_selection_rate - initial.chain_exact_selection_rate, + chain_exact_forwarding_rate: + refined.chain_exact_forwarding_rate - initial.chain_exact_forwarding_rate, + routing_success_rate: refined.routing_success_rate - initial.routing_success_rate, + }; + const regressions = Object.entries(deltas) + .filter(([, value]) => value < 0) + .map(([name, value]) => `${name} regressed by ${Math.abs(value)}`); + const reasons = [...refinedGate.reasons, ...regressions]; + return { + initial, + refined, + refined_minus_initial: deltas, + refined_not_worse: regressions.length === 0, + refined_gate: refinedGate, + passed: reasons.length === 0, + reasons, + }; +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts new file mode 100644 index 0000000000..5f9ef36fa2 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-adapters.ts @@ -0,0 +1,477 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { performance } from "node:perf_hooks"; + +import { + COMPOSITIONAL_ROUTER_MAX_SUBTASK_CHARS, + COMPOSITIONAL_ROUTER_MAX_SUBTASKS, + type DecompositionPass, + type TaskDecomposer, + type TextEmbedder, +} from "./compositional-tool-router"; + +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_MAX_OUTPUT_TOKENS = 256; +const DEFAULT_EMBEDDING_BATCH_SIZE = 128; +const DEFAULT_HASH_DIMENSIONS = 1_024; +const MAX_TEXT_CHARS = 32_768; +const MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +export const MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS = 3; +export const MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS = 300_000; +export const MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS = 600_000; + +interface JsonRecord { + [key: string]: unknown; +} + +export interface ModelUsageEvent { + operation: "decomposition" | "embedding"; + pass?: DecompositionPass; + attempt?: number; + outcome?: "completed" | "failed"; + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + duration_ms: number; +} + +interface OpenAIAdapterOptions { + baseUrl: string; + model: string; + apiKey?: string; + /** Required for non-loopback HTTPS endpoints because prompts leave the host. */ + allowRemote?: boolean; + timeoutMs?: number; + onUsage?: (event: ModelUsageEvent) => void; +} + +export interface OpenAIChatDecomposerOptions extends OpenAIAdapterOptions { + maxOutputTokens?: number; + maxAttempts?: number; + /** Endpoint-specific chat-template switch for concise output. */ + reasoningControl?: "enable_thinking_false" | "thinking_false"; + /** Request the OpenAI-compatible JSON-object response mode. */ + jsonObjectResponse?: boolean; +} + +export interface OpenAITextEmbedderOptions extends OpenAIAdapterOptions { + batchSize?: number; +} + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function positiveInteger(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new TypeError(`${label} must be a positive safe integer`); + } + return value; +} + +function boundedPositiveInteger(value: number, label: string, maximum: number): number { + const validated = positiveInteger(value, label); + if (validated > maximum) throw new RangeError(`${label} must not exceed ${maximum}`); + return validated; +} + +class ModelHttpStatusError extends Error { + constructor(readonly status: number) { + super("model request failed"); + } +} + +function endpoint( + baseUrl: string, + leaf: "chat/completions" | "embeddings", + allowRemote: boolean, +): string { + const value = new URL(baseUrl); + if (value.protocol !== "http:" && value.protocol !== "https:") { + throw new TypeError("model adapter baseUrl must use HTTP or HTTPS"); + } + if (value.username || value.password || value.search || value.hash) { + throw new TypeError("model adapter baseUrl must not contain credentials, query, or fragment"); + } + const hostname = value.hostname.replace(/^\[|\]$/gu, "").toLowerCase(); + const loopback = + hostname === "localhost" || hostname === "::1" || /^127(?:\.\d{1,3}){3}$/u.test(hostname); + if (!loopback && value.protocol !== "https:") { + throw new TypeError("remote model adapter endpoints must use HTTPS"); + } + if (!loopback && !allowRemote) { + throw new TypeError("remote model adapter endpoints require allowRemote"); + } + const path = value.pathname.replace(/\/+$/u, ""); + value.pathname = `${path.endsWith("/v1") ? path : `${path}/v1`}/${leaf}`.replace(/\/{2,}/gu, "/"); + return value.toString(); +} + +async function readBoundedJson(response: Response): Promise { + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > MAX_RESPONSE_BYTES) { + throw new Error("model adapter response is too large"); + } + if (!response.body) return JSON.parse(await response.text()) as unknown; + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > MAX_RESPONSE_BYTES) { + await reader.cancel(); + throw new Error("model adapter response is too large"); + } + chunks.push(value); + } + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return JSON.parse(new TextDecoder().decode(bytes)) as unknown; +} + +function requestHeaders(apiKey: string | undefined): Record { + return { + "content-type": "application/json", + ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}), + }; +} + +function boundedSignal(timeoutMs: number, parent?: AbortSignal): AbortSignal { + const timeout = AbortSignal.timeout(timeoutMs); + return parent ? AbortSignal.any([parent, timeout]) : timeout; +} + +function usageFrom( + value: unknown, +): Pick { + if (!isRecord(value)) return {}; + const result: Pick = {}; + for (const key of ["prompt_tokens", "completion_tokens", "total_tokens"] as const) { + const item = value[key]; + if (typeof item === "number" && Number.isSafeInteger(item) && item >= 0) result[key] = item; + } + return result; +} + +function reportUsage(callback: OpenAIAdapterOptions["onUsage"], event: ModelUsageEvent): void { + try { + callback?.(event); + } catch { + // Measurement callbacks must not change routing behavior. + } +} + +function boundedText(value: string, label: string): string { + const normalized = value.trim(); + if (!normalized || normalized.length > MAX_TEXT_CHARS) { + throw new TypeError(`${label} must contain 1-${MAX_TEXT_CHARS} characters`); + } + return normalized; +} + +function decompositionMessages( + query: string, + pass: DecompositionPass, + hints: readonly string[], + jsonObjectResponse: boolean, +): Array<{ role: "system" | "user"; content: string }> { + const system = [ + jsonObjectResponse + ? 'Return only a JSON object with one field named "subtasks", whose value is an array of strings.' + : "Return only a JSON array of strings.", + "Split the request into a short ordered list of concrete actions.", + "Each action must be solvable with one external capability.", + "Return an empty array when the request needs no external capability.", + ].join(" "); + const user = + pass === "initial" + ? `Identify the atomic actions in this request:\n${query}` + : [ + "Reconsider the action boundaries using these capability names as vocabulary hints:", + hints.join(", "), + "Request:", + query, + ].join("\n"); + return [ + { role: "system", content: system }, + { role: "user", content: user }, + ]; +} + +function jsonPayloadText(value: string): string { + const trimmed = value.trim(); + if (!trimmed.startsWith("```")) return trimmed; + const firstNewline = trimmed.indexOf("\n"); + const lastFence = trimmed.lastIndexOf("```"); + if (firstNewline < 0 || lastFence <= firstNewline) return trimmed; + return trimmed.slice(firstNewline + 1, lastFence).trim(); +} + +function parseDecompositionResponse(value: unknown): string[] { + if (!isRecord(value) || !Array.isArray(value.choices)) { + throw new Error("decomposition response has an unsupported shape"); + } + const first = value.choices[0]; + const content = isRecord(first) && isRecord(first.message) ? first.message.content : null; + if (typeof content !== "string") { + throw new Error("decomposition response has no text content"); + } + let parsed: unknown; + try { + parsed = JSON.parse(jsonPayloadText(content)); + } catch { + throw new Error("decomposition response contains invalid JSON"); + } + const subtasks = isRecord(parsed) && Array.isArray(parsed.subtasks) ? parsed.subtasks : parsed; + if (!Array.isArray(subtasks) || subtasks.length > COMPOSITIONAL_ROUTER_MAX_SUBTASKS) { + throw new Error("decomposition response has an invalid subtask array"); + } + return subtasks.map((item) => { + if (typeof item !== "string") { + throw new Error("decomposition response has a non-string subtask"); + } + const normalized = item.trim(); + if (!normalized || normalized.length > COMPOSITIONAL_ROUTER_MAX_SUBTASK_CHARS) { + throw new Error("decomposition response has an invalid subtask"); + } + return normalized; + }); +} + +function retryableDecompositionError(error: unknown, parent: AbortSignal | undefined): boolean { + if (parent?.aborted) return false; + if (!(error instanceof ModelHttpStatusError)) return true; + return error.status === 408 || error.status === 429 || error.status >= 500; +} + +/** Build the two-pass decomposer using any OpenAI-compatible chat endpoint. */ +export function createOpenAIChatTaskDecomposer( + options: OpenAIChatDecomposerOptions, +): TaskDecomposer { + if ( + options.reasoningControl !== undefined && + options.reasoningControl !== "enable_thinking_false" && + options.reasoningControl !== "thinking_false" + ) { + throw new TypeError("reasoningControl is not supported"); + } + if (options.jsonObjectResponse !== undefined && typeof options.jsonObjectResponse !== "boolean") { + throw new TypeError("jsonObjectResponse must be boolean"); + } + const target = endpoint(options.baseUrl, "chat/completions", options.allowRemote === true); + const model = boundedText(options.model, "decomposition model"); + const timeoutMs = boundedPositiveInteger( + options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + "timeoutMs", + MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS, + ); + const maxOutputTokens = positiveInteger( + options.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS, + "maxOutputTokens", + ); + const maxAttempts = boundedPositiveInteger( + options.maxAttempts ?? 1, + "maxAttempts", + MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS, + ); + if (timeoutMs * maxAttempts > MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS) { + throw new RangeError( + `timeoutMs * maxAttempts must not exceed ${MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS}`, + ); + } + return { + decompose: async (request) => { + const query = boundedText(request.query, "decomposition query"); + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const startedAt = performance.now(); + let body: unknown; + let outcome: ModelUsageEvent["outcome"] = "failed"; + try { + const response = await fetch(target, { + method: "POST", + headers: requestHeaders(options.apiKey), + body: JSON.stringify({ + model, + messages: decompositionMessages( + query, + request.pass, + request.tool_hints, + options.jsonObjectResponse === true, + ), + temperature: 0, + max_tokens: maxOutputTokens, + stream: false, + ...(options.reasoningControl === "enable_thinking_false" + ? { chat_template_kwargs: { enable_thinking: false } } + : options.reasoningControl === "thinking_false" + ? { chat_template_kwargs: { thinking: false } } + : {}), + ...(options.jsonObjectResponse ? { response_format: { type: "json_object" } } : {}), + }), + signal: boundedSignal(timeoutMs, request.signal), + }); + if (!response.ok) throw new ModelHttpStatusError(response.status); + body = await readBoundedJson(response); + const parsed = parseDecompositionResponse(body); + outcome = "completed"; + return parsed; + } catch (error) { + if (!retryableDecompositionError(error, request.signal) || attempt === maxAttempts) { + throw new Error("decomposition request failed"); + } + } finally { + // Never include prompts, endpoints, model IDs, keys, or response text. + const usage = isRecord(body) ? usageFrom(body.usage) : {}; + reportUsage(options.onUsage, { + operation: "decomposition", + pass: request.pass, + attempt, + outcome, + ...usage, + duration_ms: performance.now() - startedAt, + }); + } + } + throw new Error("decomposition request failed"); + }, + }; +} + +function parseEmbeddingResponse(value: unknown, expected: number): number[][] { + if (!isRecord(value) || !Array.isArray(value.data) || value.data.length !== expected) { + throw new Error("embedding response has an unsupported shape"); + } + const ordered = [...value.data].sort((left, right) => { + const leftIndex = isRecord(left) && typeof left.index === "number" ? left.index : -1; + const rightIndex = isRecord(right) && typeof right.index === "number" ? right.index : -1; + return leftIndex - rightIndex; + }); + return ordered.map((item, expectedIndex) => { + if (!isRecord(item) || item.index !== expectedIndex || !Array.isArray(item.embedding)) { + throw new Error("embedding response has an unsupported shape"); + } + const vector = item.embedding; + if ( + vector.length === 0 || + vector.some((number) => typeof number !== "number" || !Number.isFinite(number)) + ) { + throw new Error("embedding response has an invalid vector"); + } + return vector as number[]; + }); +} + +/** Build a dense embedder using an OpenAI-compatible embeddings endpoint. */ +export function createOpenAITextEmbedder(options: OpenAITextEmbedderOptions): TextEmbedder { + const target = endpoint(options.baseUrl, "embeddings", options.allowRemote === true); + const model = boundedText(options.model, "embedding model"); + const timeoutMs = boundedPositiveInteger( + options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + "timeoutMs", + MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS, + ); + const batchSize = positiveInteger(options.batchSize ?? DEFAULT_EMBEDDING_BATCH_SIZE, "batchSize"); + return { + embed: async (texts, signal) => { + const output: number[][] = []; + for (let start = 0; start < texts.length; start += batchSize) { + const input = texts + .slice(start, start + batchSize) + .map((text) => boundedText(text, "embedding input")); + const startedAt = performance.now(); + let body: unknown; + let outcome: ModelUsageEvent["outcome"] = "failed"; + try { + const response = await fetch(target, { + method: "POST", + headers: requestHeaders(options.apiKey), + body: JSON.stringify({ model, input }), + signal: boundedSignal(timeoutMs, signal), + }); + if (!response.ok) throw new Error("embedding request failed"); + body = await readBoundedJson(response); + output.push(...parseEmbeddingResponse(body, input.length)); + outcome = "completed"; + } catch { + throw new Error("embedding request failed"); + } finally { + const usage = isRecord(body) ? usageFrom(body.usage) : {}; + reportUsage(options.onUsage, { + operation: "embedding", + outcome, + ...usage, + duration_ms: performance.now() - startedAt, + }); + } + } + return output; + }, + }; +} + +function fnv1a(text: string): number { + let hash = 0x81_1c_9d_c5; + for (let index = 0; index < text.length; index += 1) { + hash ^= text.charCodeAt(index); + hash = Math.imul(hash, 0x01_00_01_93); + } + return hash >>> 0; +} + +function lexicalFeatures(text: string): string[] { + const words = text.toLowerCase().match(/[a-z0-9]+/gu) ?? []; + const features = [...words]; + for (let index = 0; index + 1 < words.length; index += 1) { + features.push(`${words[index]}_${words[index + 1]}`); + } + for (const word of words) { + if (word.length < 4) continue; + for (let index = 0; index + 2 < word.length; index += 1) { + features.push(`#${word.slice(index, index + 3)}`); + } + } + return features; +} + +/** + * Dependency-free lexical hashing for unit tests and portable smoke runs. + * This adapter is not a semantic model. Measured runs should use + * `createOpenAITextEmbedder` with a recorded model and immutable revision. + */ +export class PortableHashingTextEmbedder implements TextEmbedder { + readonly dimensions: number; + + constructor(dimensions = DEFAULT_HASH_DIMENSIONS) { + this.dimensions = positiveInteger(dimensions, "dimensions"); + } + + async embed( + texts: readonly string[], + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) throw new DOMException("embedding aborted", "AbortError"); + return texts.map((text) => { + const counts = new Map(); + for (const feature of lexicalFeatures(text)) { + counts.set(feature, (counts.get(feature) ?? 0) + 1); + } + const vector = Array.from({ length: this.dimensions }).fill(0); + for (const [feature, count] of counts) { + const hash = fnv1a(feature); + const index = hash % this.dimensions; + const sign = (hash & 0x80_00_00_00) === 0 ? 1 : -1; + vector[index] += sign * (1 + Math.log(count)); + } + // Empty strings are rejected by callers; keep a defined vector for text + // made only of punctuation so the router can fail safely during L2 normalization. + return vector; + }); + } +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts new file mode 100644 index 0000000000..7f1f8a6478 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-run.ts @@ -0,0 +1,435 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { performance } from "node:perf_hooks"; + +import { canonicalJson, type JsonValue, sha256Hex } from "./catalog"; +import { + buildDenseToolIndex, + COMPOSITIONAL_ROUTER_HINT_COUNT, + COMPOSITIONAL_ROUTER_TOP_K, + type CompositionalRoutingEvidence, + type CompositionalRoutingResult, + routeCompositionalToolsPaired, +} from "./compositional-tool-router"; +import { + COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + type CompositionalRoutingCasePrediction, + type CompositionalRoutingComparison, + compareCompositionalRoutingVariants, +} from "./compositional-tool-routing-acceptance"; +import { + createOpenAIChatTaskDecomposer, + createOpenAITextEmbedder, + MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS, + type ModelUsageEvent, + PortableHashingTextEmbedder, +} from "./compositional-tool-routing-adapters"; + +const SAFE_ENV_NAME = /^[A-Z][A-Z0-9_]{0,127}$/u; +const DEFAULT_RUN_TIMEOUT_MS = 15 * 60_000; +const MAX_RUN_TIMEOUT_MS = 45 * 60_000; + +export interface CompositionalRoutingRemoteModelConfig { + base_url: string; + model: string; + revision: string; + api_key_env?: string; + allow_remote?: boolean; + reasoning_control?: "enable_thinking_false" | "thinking_false"; + json_object_response?: boolean; + max_attempts?: number; +} + +export interface CompositionalRoutingAcceptanceConfig { + decomposer: CompositionalRoutingRemoteModelConfig; + embedding: + | { kind: "portable"; dimensions?: number } + | ({ kind: "openai" } & CompositionalRoutingRemoteModelConfig); + timeout_ms?: number; + run_timeout_ms?: number; +} + +export interface CompositionalRoutingAcceptanceOutput { + schema_version: "nemoclaw.compositional_tool_routing_acceptance.v1"; + generated_at: string; + claim_eligible: false; + configuration: { + decomposer_model: string; + decomposer_revision: string; + decomposer_reasoning_control: "enable_thinking_false" | "thinking_false" | "endpoint-default"; + decomposer_output_mode: "json-object" | "prompt-only"; + decomposer_max_attempts: number; + request_timeout_ms: number; + run_timeout_ms: number; + embedding_kind: "portable" | "openai"; + embedding_model: string; + embedding_revision: string; + top_k: number; + hint_count: number; + temperature: 0; + }; + corpus: { + tool_count: number; + case_count: number; + expected_step_count: number; + tools_sha256: string; + cases_sha256: string; + }; + usage: { + decomposition: { + requests: number; + failed_requests: number; + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + duration_ms: number; + }; + embedding: { + requests: number; + failed_requests: number; + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + duration_ms: number; + }; + index_build_time_ms: number; + end_to_end_time_ms: number; + }; + execution: { + status: "completed" | "timed-out"; + completed_case_count: number; + total_case_count: number; + }; + comparison: CompositionalRoutingComparison; + cases: readonly { + case_id: string; + expected_step_count: number; + evaluation_status: "completed" | "run-deadline-exceeded"; + shared_initial_decomposition: boolean; + initial: { + disposition: CompositionalRoutingResult["disposition"]; + forwarded_tool_names: readonly string[]; + forwarded_tool_count: number; + evidence: CompositionalRoutingEvidence; + }; + refined: { + disposition: CompositionalRoutingResult["disposition"]; + forwarded_tool_names: readonly string[]; + forwarded_tool_count: number; + evidence: CompositionalRoutingEvidence; + }; + }[]; + acceptance_passed: boolean; + limitations: readonly string[]; +} + +function requiredText(value: unknown, label: string): string { + if (typeof value !== "string" || !value.trim() || /[\r\n]/u.test(value)) { + throw new TypeError(`${label} must be a non-empty single-line string`); + } + return value.trim(); +} + +function publicIdentity(value: unknown, label: string): string { + const text = requiredText(value, label); + if ( + text.length > 256 || + text.includes("://") || + /(?:token|secret|password|api[_-]?key)/iu.test(text) + ) { + throw new TypeError(`${label} is not a public-safe model identity`); + } + return text; +} + +function credential(name: string | undefined, label: string): string | undefined { + if (name === undefined) return undefined; + if (!SAFE_ENV_NAME.test(name)) + throw new TypeError(`${label} must be an environment variable name`); + const value = process.env[name]; + if (!value) throw new Error(`${label} is not set`); + return value; +} + +function remoteOptions( + config: CompositionalRoutingRemoteModelConfig, + timeoutMs: number, + usage: ModelUsageEvent[], +) { + return { + baseUrl: requiredText(config.base_url, "model base_url"), + model: publicIdentity(config.model, "model"), + apiKey: credential(config.api_key_env, "model api_key_env"), + allowRemote: config.allow_remote === true, + timeoutMs, + onUsage: (event: ModelUsageEvent) => usage.push(event), + }; +} + +function prediction( + caseId: string, + result: CompositionalRoutingResult, +): CompositionalRoutingCasePrediction { + return { + case_id: caseId, + disposition: result.disposition, + fallback: result.evidence.fallback, + steps: result.evidence.final_candidate_tool_names.map((ranked, index) => ({ + subtask: `private-step-${index + 1}`, + ranked_tool_names: ranked, + })), + selected_tool_names: result.selected_tool_names, + }; +} + +function observation(result: CompositionalRoutingResult) { + const forwardedToolNames = + result.disposition === "passthrough" + ? COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.map((tool) => tool.name) + : [...result.selected_tool_names]; + return { + disposition: result.disposition, + forwarded_tool_names: forwardedToolNames, + forwarded_tool_count: forwardedToolNames.length, + evidence: result.evidence, + }; +} + +function runDeadlineResult(): CompositionalRoutingResult { + return { + disposition: "passthrough", + selected_tool_names: [], + evidence: { + initial_subtask_count: 0, + refined_subtask_count: 0, + decomposition_passes: 0, + hint_count: 0, + hint_tool_names: [], + initial_candidate_counts: [], + initial_candidate_tool_names: [], + final_candidate_counts: [], + final_candidate_tool_names: [], + selected_tool_count: 0, + selected_tool_names: [], + fallback: "run-deadline-exceeded", + timings: { + initial_decomposition_ms: 0, + catalog_embedding_ms: 0, + initial_retrieval_ms: 0, + refinement_ms: 0, + final_retrieval_ms: 0, + total_ms: 0, + }, + }, + }; +} + +function sumUsage( + events: readonly ModelUsageEvent[], + key: "prompt_tokens" | "completion_tokens" | "total_tokens", +) { + const values = events + .map((event) => event[key]) + .filter((value): value is number => value !== undefined); + return values.length === 0 ? undefined : values.reduce((total, value) => total + value, 0); +} + +function operationUsage( + events: readonly ModelUsageEvent[], + operation: ModelUsageEvent["operation"], +): CompositionalRoutingAcceptanceOutput["usage"]["decomposition"] { + const selected = events.filter((event) => event.operation === operation); + const promptTokens = sumUsage(selected, "prompt_tokens"); + const completionTokens = sumUsage(selected, "completion_tokens"); + const totalTokens = sumUsage(selected, "total_tokens"); + return { + requests: selected.length, + failed_requests: selected.filter((event) => event.outcome === "failed").length, + ...(promptTokens === undefined ? {} : { prompt_tokens: promptTokens }), + ...(completionTokens === undefined ? {} : { completion_tokens: completionTokens }), + ...(totalTokens === undefined ? {} : { total_tokens: totalTokens }), + duration_ms: selected.reduce((total, event) => total + event.duration_ms, 0), + }; +} + +/** Run the public-safe route-only acceptance path with a real decomposer. */ +export async function runCompositionalRoutingAcceptance( + config: CompositionalRoutingAcceptanceConfig, +): Promise { + const timeoutMs = config.timeout_ms ?? 120_000; + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs <= 0 || + timeoutMs > MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS + ) { + throw new TypeError( + `timeout_ms must be a positive safe integer no greater than ${MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS}`, + ); + } + const runTimeoutMs = config.run_timeout_ms ?? DEFAULT_RUN_TIMEOUT_MS; + if ( + !Number.isSafeInteger(runTimeoutMs) || + runTimeoutMs <= 0 || + runTimeoutMs > MAX_RUN_TIMEOUT_MS + ) { + throw new TypeError( + `run_timeout_ms must be a positive safe integer no greater than ${MAX_RUN_TIMEOUT_MS}`, + ); + } + const usage: ModelUsageEvent[] = []; + const decomposerRevision = publicIdentity(config.decomposer.revision, "decomposer revision"); + const decomposerMaxAttempts = config.decomposer.max_attempts ?? 1; + const decomposer = createOpenAIChatTaskDecomposer({ + ...remoteOptions(config.decomposer, timeoutMs, usage), + reasoningControl: config.decomposer.reasoning_control, + jsonObjectResponse: config.decomposer.json_object_response, + maxAttempts: decomposerMaxAttempts, + }); + const embeddingKind = config.embedding.kind; + const embeddingModel = + embeddingKind === "portable" + ? "portable-lexical-hashing" + : publicIdentity(config.embedding.model, "embedding model"); + const embeddingRevision = + embeddingKind === "portable" + ? "builtin-v1" + : publicIdentity(config.embedding.revision, "embedding revision"); + const embedder = + embeddingKind === "portable" + ? new PortableHashingTextEmbedder(config.embedding.dimensions) + : createOpenAITextEmbedder(remoteOptions(config.embedding, timeoutMs, usage)); + + const startedAt = performance.now(); + const deadlineAt = startedAt + runTimeoutMs; + const runSignal = AbortSignal.timeout(runTimeoutMs); + const deadlineReached = () => runSignal.aborted || performance.now() >= deadlineAt; + const indexStartedAt = performance.now(); + let preparedIndex: Awaited> | null = null; + let indexError: unknown; + let indexFailed = false; + try { + preparedIndex = await buildDenseToolIndex( + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + embedder, + runSignal, + ); + } catch (error) { + indexFailed = true; + indexError = error; + } + const indexBuildTimeMs = performance.now() - indexStartedAt; + let runTimedOut = deadlineReached(); + if (indexFailed && !runTimedOut) throw indexError; + const initialPredictions: CompositionalRoutingCasePrediction[] = []; + const refinedPredictions: CompositionalRoutingCasePrediction[] = []; + const cases: CompositionalRoutingAcceptanceOutput["cases"][number][] = []; + let completedCaseCount = 0; + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + runTimedOut ||= deadlineReached(); + const attemptedPair = + runTimedOut || preparedIndex === null + ? null + : await routeCompositionalToolsPaired( + fixture.prompt, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + { decomposer, embedder, preparedIndex, signal: runSignal }, + ); + runTimedOut ||= deadlineReached(); + const pair = + runTimedOut || attemptedPair === null + ? { + initial: runDeadlineResult(), + refined: runDeadlineResult(), + shared_initial_decomposition: false as const, + } + : attemptedPair; + if (!runTimedOut) completedCaseCount += 1; + initialPredictions.push(prediction(fixture.id, pair.initial)); + refinedPredictions.push(prediction(fixture.id, pair.refined)); + cases.push({ + case_id: fixture.id, + expected_step_count: fixture.expected_steps.length, + evaluation_status: runTimedOut ? "run-deadline-exceeded" : "completed", + shared_initial_decomposition: pair.shared_initial_decomposition, + initial: observation(pair.initial), + refined: observation(pair.refined), + }); + } + runTimedOut ||= deadlineReached(); + const evaluatedComparison = compareCompositionalRoutingVariants( + { variant: "initial", cases: initialPredictions }, + { variant: "refined", cases: refinedPredictions }, + ); + runTimedOut ||= deadlineReached(); + const comparison: CompositionalRoutingComparison = runTimedOut + ? { + ...evaluatedComparison, + passed: false, + reasons: [ + ...evaluatedComparison.reasons, + `run deadline exceeded after ${completedCaseCount} of ${COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.length} cases`, + ], + } + : evaluatedComparison; + return { + schema_version: "nemoclaw.compositional_tool_routing_acceptance.v1", + generated_at: new Date().toISOString(), + claim_eligible: false, + configuration: { + decomposer_model: publicIdentity(config.decomposer.model, "decomposer model"), + decomposer_revision: decomposerRevision, + decomposer_reasoning_control: config.decomposer.reasoning_control ?? "endpoint-default", + decomposer_output_mode: + config.decomposer.json_object_response === true ? "json-object" : "prompt-only", + decomposer_max_attempts: decomposerMaxAttempts, + request_timeout_ms: timeoutMs, + run_timeout_ms: runTimeoutMs, + embedding_kind: embeddingKind, + embedding_model: embeddingModel, + embedding_revision: embeddingRevision, + top_k: COMPOSITIONAL_ROUTER_TOP_K, + hint_count: COMPOSITIONAL_ROUTER_HINT_COUNT, + temperature: 0, + }, + corpus: { + tool_count: COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.length, + case_count: COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.length, + expected_step_count: COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.reduce( + (total, fixture) => total + fixture.expected_steps.length, + 0, + ), + tools_sha256: sha256Hex( + canonicalJson(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS as unknown as JsonValue), + ), + cases_sha256: sha256Hex( + canonicalJson(COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES as unknown as JsonValue), + ), + }, + usage: { + decomposition: operationUsage(usage, "decomposition"), + embedding: operationUsage(usage, "embedding"), + index_build_time_ms: indexBuildTimeMs, + end_to_end_time_ms: performance.now() - startedAt, + }, + execution: { + status: runTimedOut ? "timed-out" : "completed", + completed_case_count: completedCaseCount, + total_case_count: COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.length, + }, + comparison, + cases, + acceptance_passed: !runTimedOut && comparison.passed, + limitations: [ + "This route-only acceptance run does not execute an agent or a tool.", + "The eight-case corpus is software acceptance coverage, not a universal quality result.", + ...(decomposerRevision === "unreported" + ? ["The decomposer revision was not reported, so this run is not reproducible evidence."] + : []), + ...(embeddingKind === "portable" + ? ["The portable lexical embedder is for smoke testing and is not claim eligible."] + : []), + ], + }; +} diff --git a/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts b/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts new file mode 100644 index 0000000000..91679b3441 --- /dev/null +++ b/scripts/performance/tool-disclosure/compositional-tool-routing-transform.ts @@ -0,0 +1,389 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { + buildDenseToolIndex, + type CompositionalRouterOptions, + type CompositionalRoutingEvidence, + type DenseToolIndex, + routeCompositionalTools, + type ToolRoutingCatalogEntry, +} from "./compositional-tool-router"; +import type { + RecorderEndpoint, + RecorderMethod, + RecordingProxyRequestTransform, + RecordingProxyTransformInput, +} from "./recorder"; + +const SAFE_TOOL_NAME = /^[A-Za-z0-9_-]{1,128}$/u; + +interface JsonRecord { + [key: string]: unknown; +} + +interface ParsedTool { + name: string; + description: string; + definition: unknown; + routable: boolean; +} + +interface CachedRoute { + selectedNames: ReadonlySet; + evidence: CompositionalTransformEvidence; +} + +interface CachedRouteWork { + promise: Promise; + controller: AbortController; + waiterCount: number; + settled: boolean; +} + +export interface CompositionalTransformEvidence { + run_id: string; + source_tool_count: number; + routable_tool_count: number; + preserved_tool_count: number; + forwarded_tool_count: number; + cache_hits: number; + index_cache_hit: boolean; + transform_bypass: "tool-choice-conflict" | null; + routing: CompositionalRoutingEvidence; +} + +export interface CompositionalToolRoutingTransformOptions extends CompositionalRouterOptions { + /** + * Explicitly identify the reviewed catalog under test while retaining every + * framework, provider-native, and core tool. + */ + isRoutableTool: (name: string) => boolean; + /** Make unsupported/bypassed requests fail instead of silently acting as direct mode. */ + requireRouting?: boolean; +} + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function parsedTool( + definition: unknown, + isRoutableTool: (name: string) => boolean, +): ParsedTool | null { + if (!isRecord(definition) || definition.type !== "function") return null; + const nested = isRecord(definition.function) ? definition.function : null; + if (!nested) return null; + const rawName = nested.name; + const rawDescription = nested.description ?? ""; + if ( + typeof rawName !== "string" || + !SAFE_TOOL_NAME.test(rawName) || + typeof rawDescription !== "string" + ) { + return null; + } + return { + name: rawName, + description: rawDescription, + definition, + routable: isRoutableTool(rawName), + }; +} + +function textContent(value: unknown): string | null { + if (typeof value === "string" && value.trim()) return value; + if (!Array.isArray(value)) return null; + const parts = value.flatMap((part) => { + if (!isRecord(part)) return []; + const text = part.text; + return typeof text === "string" && text.trim() ? [text] : []; + }); + return parts.length > 0 ? parts.join("\n") : null; +} + +function extractChatQuery(payload: JsonRecord): string | null { + if (!Array.isArray(payload.messages)) return null; + for (let index = payload.messages.length - 1; index >= 0; index -= 1) { + const message = payload.messages[index]; + if (!isRecord(message) || message.role !== "user") continue; + const content = textContent(message.content); + if (content) return content; + } + return null; +} + +function catalogFingerprint(tools: readonly ParsedTool[]): string { + const hash = createHash("sha256"); + for (const tool of tools) { + hash.update(tool.name, "utf8"); + hash.update("\0", "utf8"); + hash.update(tool.description, "utf8"); + hash.update("\0", "utf8"); + hash.update(tool.routable ? "route" : "preserve", "utf8"); + hash.update("\0", "utf8"); + } + return hash.digest("hex"); +} + +function queryFingerprint(query: string): string { + return createHash("sha256").update(query, "utf8").digest("hex"); +} + +function namedToolChoice(payload: JsonRecord): string | null { + if (!isRecord(payload.tool_choice) || !isRecord(payload.tool_choice.function)) return null; + const name = payload.tool_choice.function.name; + return typeof name === "string" ? name : null; +} + +function cloneEvidence(value: CompositionalTransformEvidence): CompositionalTransformEvidence { + return structuredClone(value); +} + +function abortError(): DOMException { + return new DOMException("routing aborted", "AbortError"); +} + +/** + * Let each request stop waiting independently without cancelling shared index + * or route work that another live request may still need. + */ +function awaitWithAbort(pending: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(abortError()); + return new Promise((resolve, reject) => { + const onAbort = () => reject(abortError()); + signal.addEventListener("abort", onAbort, { once: true }); + pending.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (error: unknown) => { + signal.removeEventListener("abort", onAbort); + reject(error); + }, + ); + }); +} + +/** + * Connect the pure refined router to the recording proxy without changing + * an agent's executor registry. Only the forwarded model request is filtered. + */ +export class CompositionalToolRoutingTransform { + readonly requestTransform: RecordingProxyRequestTransform; + + private readonly isRoutableTool: (name: string) => boolean; + private readonly routes = new Map(); + private readonly routeKeysByRun = new Map>(); + private readonly indexes = new Map>(); + + constructor(private readonly options: CompositionalToolRoutingTransformOptions) { + if (typeof options.isRoutableTool !== "function") { + throw new TypeError("compositional routing requires an explicit isRoutableTool predicate"); + } + this.isRoutableTool = options.isRoutableTool; + this.requestTransform = (input) => this.transform(input); + } + + async consumeEvidence(runId: string): Promise { + const keys = this.routeKeysByRun.get(runId); + if (!keys) return []; + this.routeKeysByRun.delete(runId); + const evidence: CompositionalTransformEvidence[] = []; + for (const key of keys) { + const work = this.routes.get(key); + this.routes.delete(key); + if (!work) continue; + try { + evidence.push(cloneEvidence((await work.promise).evidence)); + } catch { + // Aborted/failed routes do not retain partial evidence. + } + } + return evidence; + } + + private bypass(body: Buffer, reason: string): Buffer { + if (this.options.requireRouting) { + throw new Error(`required compositional routing bypassed: ${reason}`); + } + return body; + } + + private evictRoute(runId: string, routeKey: string, work: CachedRouteWork): void { + if (this.routes.get(routeKey) === work) this.routes.delete(routeKey); + const runKeys = this.routeKeysByRun.get(runId); + runKeys?.delete(routeKey); + if (runKeys?.size === 0) this.routeKeysByRun.delete(runId); + } + + private preparedIndex( + fingerprint: string, + catalog: readonly ToolRoutingCatalogEntry[], + ): { promise: Promise; hit: boolean } { + const existing = this.indexes.get(fingerprint); + if (existing) return { promise: existing, hit: true }; + // Cached work must not inherit one request's cancellation signal. + const promise = buildDenseToolIndex(catalog, this.options.embedder); + this.indexes.set(fingerprint, promise); + void promise.catch(() => { + if (this.indexes.get(fingerprint) === promise) this.indexes.delete(fingerprint); + }); + if (this.indexes.size > 32) { + const oldest = this.indexes.keys().next().value as string | undefined; + if (oldest && oldest !== fingerprint) this.indexes.delete(oldest); + } + return { promise, hit: false }; + } + + private shouldTransform( + endpoint: RecorderEndpoint, + method: RecorderMethod, + modelCallSequence: number | null, + runId: string | null, + ): runId is string { + return ( + endpoint === "chat-completions" && + method === "POST" && + modelCallSequence !== null && + runId !== null + ); + } + + private async transform(input: RecordingProxyTransformInput): Promise { + if (!this.shouldTransform(input.endpoint, input.method, input.modelCallSequence, input.runId)) { + return this.bypass(input.body, "unsupported request boundary"); + } + const runId = input.runId; + + let payload: JsonRecord; + try { + const value: unknown = JSON.parse(input.body.toString("utf8")); + if (!isRecord(value)) return this.bypass(input.body, "request body is not an object"); + payload = value; + } catch { + return this.bypass(input.body, "request body is not valid JSON"); + } + if (!Array.isArray(payload.tools) || payload.tools.length === 0) { + return this.bypass(input.body, "request has no tools"); + } + + const parsed = payload.tools.map((tool) => parsedTool(tool, this.isRoutableTool)); + // Preserve the whole request when any schema shape is unknown; never drop a + // tool the transformer cannot identify safely. + if (parsed.some((tool) => tool === null)) { + return this.bypass(input.body, "request contains an unsupported tool schema"); + } + const tools = parsed as ParsedTool[]; + const routable = tools.filter((tool) => tool.routable); + if (routable.length === 0) return this.bypass(input.body, "request has no routable tools"); + + const fingerprint = catalogFingerprint(tools); + const query = extractChatQuery(payload); + if (!query) return this.bypass(input.body, "request has no user query"); + const routeKey = `${runId}:${queryFingerprint(query)}:${fingerprint}`; + let work = this.routes.get(routeKey); + const cacheHit = work !== undefined; + + if (!work) { + const catalog: ToolRoutingCatalogEntry[] = routable.map((tool) => ({ + name: tool.name, + description: tool.description, + definition: tool.definition, + })); + const prepared = this.preparedIndex(fingerprint, catalog); + const controller = new AbortController(); + const routingSignal = this.options.signal + ? AbortSignal.any([this.options.signal, controller.signal]) + : controller.signal; + const promise = (async (): Promise => { + const result = await routeCompositionalTools(query, catalog, { + ...this.options, + preparedIndex: prepared.promise, + signal: routingSignal, + }); + const selected = + result.disposition === "passthrough" + ? new Set(routable.map((tool) => tool.name)) + : new Set(result.selected_tool_names); + if ( + result.selected_tool_names.some((name) => !routable.some((tool) => tool.name === name)) + ) { + throw new Error("router selected a tool outside the reviewed catalog"); + } + const forwardedToolCount = tools.filter( + (tool) => !tool.routable || selected.has(tool.name), + ).length; + return { + selectedNames: selected, + evidence: { + run_id: runId, + source_tool_count: tools.length, + routable_tool_count: routable.length, + preserved_tool_count: tools.length - routable.length, + forwarded_tool_count: + result.disposition === "passthrough" ? tools.length : forwardedToolCount, + cache_hits: 0, + index_cache_hit: prepared.hit, + transform_bypass: null, + routing: result.evidence, + }, + }; + })(); + work = { promise, controller, waiterCount: 0, settled: false }; + this.routes.set(routeKey, work); + const runKeys = this.routeKeysByRun.get(runId) ?? new Set(); + runKeys.add(routeKey); + this.routeKeysByRun.set(runId, runKeys); + const createdWork = work; + void promise.then( + () => { + createdWork.settled = true; + }, + () => { + createdWork.settled = true; + this.evictRoute(runId, routeKey, createdWork); + }, + ); + } + + work.waiterCount += 1; + let cached: CachedRoute; + try { + cached = await awaitWithAbort(work.promise, input.signal); + } finally { + work.waiterCount -= 1; + if (work.waiterCount === 0 && !work.settled) { + work.controller.abort(); + this.evictRoute(runId, routeKey, work); + } + } + if (cacheHit) cached.evidence.cache_hits += 1; + + if (cached.evidence.routing.fallback !== null) { + return this.bypass(input.body, `routing fallback: ${cached.evidence.routing.fallback}`); + } + const choice = namedToolChoice(payload); + if ( + choice && + routable.some((tool) => tool.name === choice) && + !cached.selectedNames.has(choice) + ) { + cached.evidence.transform_bypass = "tool-choice-conflict"; + cached.evidence.forwarded_tool_count = tools.length; + return this.bypass(input.body, "named tool choice conflicts with routed catalog"); + } + const forwardedTools = tools.filter( + (tool) => !tool.routable || cached.selectedNames.has(tool.name), + ); + if (payload.tool_choice === "required" && forwardedTools.length === 0) { + cached.evidence.transform_bypass = "tool-choice-conflict"; + cached.evidence.forwarded_tool_count = tools.length; + return this.bypass(input.body, "required tool choice has no routed tools"); + } + payload.tools = forwardedTools.map((tool) => tool.definition); + return Buffer.from(JSON.stringify(payload), "utf8"); + } +} diff --git a/scripts/performance/tool-disclosure/drivers.ts b/scripts/performance/tool-disclosure/drivers.ts new file mode 100644 index 0000000000..f417e6d921 --- /dev/null +++ b/scripts/performance/tool-disclosure/drivers.ts @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; + +import type { ToolDisclosureAgent } from "./schedule"; + +export interface AgentDriverCommand { + command: string; + args: string[]; + redactions: string[]; +} + +export const OPENCLAW_PERFORMANCE_TEST_CALLS_PATH = + "/sandbox/.nemoclaw-performance-test/calls.jsonl"; + +function validateIdentifier(value: string, label: string): void { + if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/u.test(value)) { + throw new Error(`${label} contains unsupported characters`); + } +} + +function base64(value: string): string { + return Buffer.from(value, "utf8").toString("base64"); +} + +function shellScriptFor(agent: ToolDisclosureAgent, prompt: string, sessionId: string): string { + const encodedPrompt = base64(prompt); + if (agent === "openclaw") { + return [ + "set -eu", + `prompt=$(printf '%s' '${encodedPrompt}' | base64 -d)`, + `exec openclaw agent --agent main --json --thinking off --session-id '${sessionId}' -m "$prompt"`, + ].join("; "); + } + if (agent === "langchain-deepagents-code") { + return [ + "set -eu", + `prompt=$(printf '%s' '${encodedPrompt}' | base64 -d)`, + 'exec nemoclaw-start dcode -n "$prompt"', + ].join("; "); + } + const payload = base64( + JSON.stringify({ + messages: [{ role: "user", content: prompt }], + max_tokens: 512, + temperature: 0, + stream: false, + }), + ); + return [ + "set -eu", + "set -a", + "[ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env", + "set +a", + `payload='${payload}'`, + "if [ -n \"${API_SERVER_KEY:-}\" ]; then printf '%s' \"$payload\" | base64 -d | curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer ${API_SERVER_KEY}\" --data-binary @-; else printf '%s' \"$payload\" | base64 -d | curl -fsS --max-time 600 http://127.0.0.1:8642/v1/chat/completions -H 'Content-Type: application/json' --data-binary @-; fi", + ].join("; "); +} + +export function buildAgentDriverCommand(options: { + openshellBin?: string; + sandboxName: string; + agent: ToolDisclosureAgent; + prompt: string; + sessionId: string; +}): AgentDriverCommand { + validateIdentifier(options.sandboxName, "sandboxName"); + validateIdentifier(options.sessionId, "sessionId"); + if (!options.prompt.trim()) throw new Error("performance test task prompt must not be empty"); + const script = shellScriptFor(options.agent, options.prompt, options.sessionId); + if (/[\r\n]/u.test(script)) { + throw new Error("performance test agent driver must be a single-line shell command"); + } + return { + command: options.openshellBin ?? "openshell", + args: ["sandbox", "exec", "-n", options.sandboxName, "--", "sh", "-lc", script], + redactions: [ + options.prompt, + base64(options.prompt), + base64( + JSON.stringify({ + messages: [{ role: "user", content: options.prompt }], + max_tokens: 512, + temperature: 0, + stream: false, + }), + ), + ], + }; +} + +export function outputContainsOracle(output: string, oracle: string): boolean { + return oracle.length > 0 && output.includes(oracle); +} + +function jsonDocuments(raw: string): unknown[] { + try { + const parsed = JSON.parse(raw) as unknown; + return Array.isArray(parsed) ? parsed : [parsed]; + } catch { + return raw.split(/\r?\n/u).flatMap((line) => { + try { + return [JSON.parse(line) as unknown]; + } catch { + return []; + } + }); + } +} + +function record(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function stripTerminalControls(value: string): string { + return value.replace(/\u001b\[[0-?]*[ -/]*[@-~]/gu, ""); +} + +/** + * Deep Agents Code prints the final assistant block immediately before its + * `Task completed` marker, followed by an `Agent active` timing line. Tool + * trace lines precede that block. Select only the delimited assistant block so + * neither a tool trace nor a trailing status line can satisfy an oracle. + */ +function extractDeepAgentsFinalAssistantOutput(raw: string): string { + const lines = stripTerminalControls(raw).split(/\r?\n/u); + let completed = -1; + for (let index = lines.length - 1; index >= 0; index -= 1) { + if (/^(?:[✓✔]\s*)?Task completed\b/iu.test(lines[index].trim())) { + completed = index; + break; + } + } + if (completed < 0) return ""; + + let boundary = -1; + for (let index = 0; index < completed; index += 1) { + const line = lines[index].trim(); + if (/^(?:🔧\s*)?Calling tool:/u.test(line) || /^(?:[✓✔]\s*)?Server ready\s*$/iu.test(line)) { + boundary = index; + } + } + if (boundary < 0) return ""; + return lines + .slice(boundary + 1, completed) + .join("\n") + .trim(); +} + +/** Extract only the user-visible final assistant field, never raw tool traces. */ +export function extractFinalAssistantOutput(agent: ToolDisclosureAgent, raw: string): string { + if (agent === "hermes") { + const root = record(jsonDocuments(raw)[0]); + const choice = Array.isArray(root?.choices) ? record(root.choices[0]) : null; + const message = record(choice?.message); + return typeof message?.content === "string" ? message.content : ""; + } + if (agent === "openclaw") { + const parts: string[] = []; + for (const document of jsonDocuments(raw)) { + const root = record(document); + const result = record(root?.result); + const payloads = Array.isArray(result?.payloads) + ? result.payloads + : Array.isArray(root?.payloads) + ? root.payloads + : []; + for (const payload of payloads) { + const text = record(payload)?.text; + if (typeof text === "string") parts.push(text); + } + } + return parts.join("\n"); + } + return extractDeepAgentsFinalAssistantOutput(raw); +} + +export function buildOpenClawCallLogCommand(options: { + openshellBin?: string; + sandboxName: string; + action: "read" | "reset"; +}): AgentDriverCommand { + validateIdentifier(options.sandboxName, "sandboxName"); + const script = + options.action === "reset" + ? `rm -f -- '${OPENCLAW_PERFORMANCE_TEST_CALLS_PATH}'` + : `[ ! -f '${OPENCLAW_PERFORMANCE_TEST_CALLS_PATH}' ] || cat -- '${OPENCLAW_PERFORMANCE_TEST_CALLS_PATH}'`; + return { + command: options.openshellBin ?? "openshell", + args: ["sandbox", "exec", "-n", options.sandboxName, "--", "sh", "-lc", script], + redactions: [], + }; +} diff --git a/scripts/performance/tool-disclosure/execute.ts b/scripts/performance/tool-disclosure/execute.ts new file mode 100644 index 0000000000..9d2f495e61 --- /dev/null +++ b/scripts/performance/tool-disclosure/execute.ts @@ -0,0 +1,860 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; + +import { appendJsonLine, artifactPath, writeJsonArtifact, writeTextArtifact } from "./artifacts"; +import { assembleToolDisclosureRun } from "./assemble-run"; +import { generateCatalogPrefix, type SyntheticCatalog } from "./catalog"; +import { + buildAgentDriverCommand, + buildOpenClawCallLogCommand, + extractFinalAssistantOutput, +} from "./drivers"; +import type { RecordedSyntheticCall } from "./grading"; +import { SyntheticMcpServer } from "./mcp-server"; +import { type QuickTunnel, startQuickTunnel } from "./quick-tunnel"; +import { createToolDisclosureRecordingProxy, type ToolDisclosureRecordingEvent } from "./recorder"; +import type { ScheduledToolDisclosureRun, ToolDisclosureAgent } from "./schedule"; +import { runBoundedCommand } from "./subprocess"; +import type { SyntheticPerformanceTask, SyntheticTaskSet } from "./tasks"; +import { + countTokensWithVllm, + readVllmProcessStartTime, + readVllmTokenSnapshot, + tokenDelta, +} from "./telemetry"; +import type { InferenceConfiguration, ToolDisclosureManifest, ToolDisclosureRun } from "./types"; + +export interface SanitizedRunEvidence { + run_id: string; + recorder_events: readonly ToolDisclosureRecordingEvent[]; + calls: readonly RecordedSyntheticCall[]; + invocation: { exit_code: number | null; timed_out: boolean; elapsed_ms: number }; + initial_schema_tokens: number; + prompt_tokens?: number; + completion_tokens?: number; + final_oracles_present: boolean; + failure_outcome?: "setup-error" | "context-overflow"; +} + +export interface AttemptJournalEntry { + raw: SanitizedRunEvidence; + run: ToolDisclosureRun; +} + +/** Permit retries only while an attempt is still wholly inside setup. */ +export function assertSetupRetryAllowed( + invocationAttempted: boolean, + runId: string, + cause?: unknown, +): void { + if (invocationAttempted) { + throw new Error( + `run ${runId} failed after agent invocation began; discard this campaign and restart it with fresh resources`, + { cause }, + ); + } +} + +/** Continue setup retries across process restarts without resetting the allowance. */ +export function nextSetupAttempt( + entries: readonly AttemptJournalEntry[], + runId: string, + retrySetupFailures: number, +): number { + if (!Number.isSafeInteger(retrySetupFailures) || retrySetupFailures < 0) { + throw new Error("retry_setup_failures must be a non-negative integer"); + } + const used = entries.filter( + (entry) => entry.run.run_id === runId && entry.run.outcome === "setup-error", + ).length; + if (used > retrySetupFailures) throw new Error(`run ${runId} exhausted setup retries`); + return used; +} + +export function recoverAttemptJournal(outputDir: string): AttemptJournalEntry[] { + const file = artifactPath(outputDir, "attempt-journal.jsonl"); + if (!fs.existsSync(file)) return []; + const raw = fs.readFileSync(file, "utf8"); + const lines = raw.split("\n"); + const entries: AttemptJournalEntry[] = []; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + if (!line.trim()) continue; + try { + entries.push(JSON.parse(line) as AttemptJournalEntry); + } catch { + const isUnterminatedFinalAppend = index === lines.length - 1 && !raw.endsWith("\n"); + if (!isUnterminatedFinalAppend) { + throw new Error(`attempt journal is corrupt at line ${index + 1}`); + } + writeTextArtifact( + outputDir, + "attempt-journal.jsonl", + entries.map((entry) => JSON.stringify(entry)).join("\n") + (entries.length ? "\n" : ""), + ); + break; + } + } + return entries; +} + +export function materializeAttemptJournal( + outputDir: string, + entries: readonly AttemptJournalEntry[], +): void { + const lines = (select: (entry: AttemptJournalEntry) => unknown): string => + entries.map((entry) => JSON.stringify(select(entry))).join("\n") + (entries.length ? "\n" : ""); + writeTextArtifact( + outputDir, + "raw-events.jsonl", + lines((entry) => entry.raw), + ); + writeTextArtifact( + outputDir, + "runs.jsonl", + lines((entry) => entry.run), + ); +} + +export interface LiveCampaignConfiguration { + campaign: 1 | 2; + upstream_vllm_url: string; + telemetry_url: string; + tokenizer_model: string; + recorder_port: number; + managed_inference_base_url: string; + vllm_container_name: string; + vllm_container_id: string; + sandbox_names: Record; + /** Private live sandbox instance IDs, verified against status and published only as hashes. */ + sandbox_instance_ids: Record; + sandbox_container_names: Record; + timeout_ms?: number; + openshell_bin?: string; + nemoclaw_bin?: string; + cloudflared_bin?: string; + docker_bin?: string; +} + +export interface CampaignAttestation { + campaign_id: string; + vllm_process_start_time_seconds: number; + inference_container_id_sha256: string; + inference_config_sha256: string; + inference_image_digest: string; + sandbox_cells: readonly { + cell: string; + instance_id_sha256: string; + status_sha256: string; + image_digest: string; + }[]; +} + +function sha256(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +/** Accept only the single inspect result whose immutable ID was requested. */ +export function selectInspectedContainer( + value: unknown, + expectedId: string, +): (T & { Id: string }) | undefined { + if (!Array.isArray(value) || value.length !== 1) return undefined; + return ( + (value as Array<(T & { Id: string }) | null>).find( + (candidate) => candidate?.Id === expectedId, + ) ?? undefined + ); +} + +const PUBLIC_VLLM_ENV_NAMES = new Set([ + "VLLM_ATTENTION_BACKEND", + "VLLM_USE_V1", + "VLLM_WORKER_MULTIPROC_METHOD", +]); +const SHELL_EXECUTABLE_NAMES = new Set(["ash", "bash", "dash", "ksh", "sh", "zsh"]); +const SHELL_CONTROL_TOKENS = new Set(["&", "&&", ";", "|", "||"]); + +interface InspectedVllmContainer { + Id?: string; + Image?: string; + Config?: { + Cmd?: unknown; + Entrypoint?: unknown; + Env?: unknown; + Labels?: unknown; + }; +} + +export interface AttestedVllmConfiguration { + cmd: readonly string[]; + entrypoint: readonly string[]; + env: readonly string[]; +} + +function inspectedStringArray(value: unknown): string[] | undefined { + if (value === undefined || value === null) return []; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) return undefined; + return value; +} + +/** Tokenize the literal shell command used by Docker's `sh -c` form without expanding it. */ +function tokenizeShellCommand(command: string): string[] | undefined { + const tokens: string[] = []; + let word = ""; + let wordStarted = false; + let quote: "single" | "double" | undefined; + const pushWord = (): void => { + if (!wordStarted) return; + tokens.push(word); + word = ""; + wordStarted = false; + }; + + for (let index = 0; index < command.length; index += 1) { + const character = command[index]; + if (quote === "single") { + if (character === "'") quote = undefined; + else word += character; + continue; + } + if (quote === "double") { + if (character === '"') { + quote = undefined; + } else if (character === "\\") { + index += 1; + if (index >= command.length) return undefined; + word += command[index]; + } else { + word += character; + } + continue; + } + if (/\s/u.test(character)) { + pushWord(); + if (character === "\n") tokens.push(";"); + } else if (character === "'" || character === '"') { + quote = character === "'" ? "single" : "double"; + wordStarted = true; + } else if (character === "\\") { + index += 1; + if (index >= command.length) return undefined; + word += command[index]; + wordStarted = true; + } else if (character === ";" || character === "&" || character === "|") { + pushWord(); + const doubled = command[index + 1] === character; + tokens.push(doubled ? `${character}${character}` : character); + if (doubled) index += 1; + } else { + word += character; + wordStarted = true; + } + } + if (quote) return undefined; + pushWord(); + return tokens; +} + +function inspectedProcessTokens( + entrypoint: readonly string[], + cmd: readonly string[], +): string[] | undefined { + const argv = [...entrypoint, ...cmd]; + const shellIndex = argv.findIndex((value) => + SHELL_EXECUTABLE_NAMES.has(value.split("/").pop() ?? ""), + ); + if (shellIndex < 0) return argv; + const shellOption = argv[shellIndex + 1]; + if (!shellOption?.startsWith("-") || !shellOption.slice(1).includes("c")) return argv; + const commandIndex = shellIndex + 2; + const shellTokens = tokenizeShellCommand(argv[commandIndex] ?? ""); + if (!shellTokens) return undefined; + return [...argv.slice(0, commandIndex), ...shellTokens, ...argv.slice(commandIndex + 1)]; +} + +function optionMatches(args: readonly string[], name: string, value: string): boolean { + return args.some( + (argument, index) => + argument === `${name}=${value}` || (argument === name && args[index + 1] === value), + ); +} + +function containsSequence(values: readonly string[], expected: readonly string[]): boolean { + if (expected.length === 0) return false; + return values.some((_, index) => + expected.every((value, offset) => values[index + offset] === value), + ); +} + +function vllmInvocation( + tokens: readonly string[], +): { kind: "serve" | "api-server"; args: readonly string[] } | undefined { + const candidates: { kind: "serve" | "api-server"; start: number }[] = []; + for (let index = 0; index < tokens.length; index += 1) { + const executable = tokens[index].split("/").pop() ?? ""; + if (executable === "vllm" && tokens[index + 1] === "serve") { + candidates.push({ kind: "serve", start: index + 2 }); + } + if ( + /^python(?:\d+(?:\.\d+)*)?$/u.test(executable) && + tokens[index + 1] === "-m" && + tokens[index + 2] === "vllm.entrypoints.openai.api_server" + ) { + candidates.push({ kind: "api-server", start: index + 3 }); + } + } + if (candidates.length !== 1) return undefined; + const candidate = candidates[0]; + const boundary = tokens.findIndex( + (token, index) => index >= candidate.start && SHELL_CONTROL_TOKENS.has(token), + ); + return { + kind: candidate.kind, + args: tokens.slice(candidate.start, boundary < 0 ? undefined : boundary), + }; +} + +/** Extract only a structurally verified process configuration from Docker inspect. */ +export function readAttestedVllmConfiguration( + container: InspectedVllmContainer, + expected: InferenceConfiguration, +): AttestedVllmConfiguration | undefined { + const cmd = inspectedStringArray(container.Config?.Cmd); + const entrypoint = inspectedStringArray(container.Config?.Entrypoint); + const rawEnv = inspectedStringArray(container.Config?.Env); + if (!cmd || !entrypoint || !rawEnv) return undefined; + const tokens = inspectedProcessTokens(entrypoint, cmd); + if (!tokens) return undefined; + const invocation = vllmInvocation(tokens); + if (!invocation) return undefined; + const modelMatches = + (invocation.kind === "serve" && invocation.args[0] === expected.model_id) || + optionMatches(invocation.args, "--model", expected.model_id); + const publicFlagsMatch = expected.public_vllm_flags.every((flag) => { + const flagTokens = tokenizeShellCommand(flag); + return ( + flagTokens !== undefined && + flagTokens.every((token) => !SHELL_CONTROL_TOKENS.has(token)) && + containsSequence(invocation.args, flagTokens) + ); + }); + if ( + !modelMatches || + !optionMatches(invocation.args, "--revision", expected.model_revision) || + !optionMatches(invocation.args, "--tool-call-parser", expected.tool_call_parser) || + !optionMatches(invocation.args, "--reasoning-parser", expected.reasoning_parser) || + !publicFlagsMatch + ) { + return undefined; + } + + const publicEnv = rawEnv + .filter((entry) => PUBLIC_VLLM_ENV_NAMES.has(entry.slice(0, entry.indexOf("=")))) + .sort(); + if ( + new Set(publicEnv.map((entry) => entry.slice(0, entry.indexOf("=")))).size !== publicEnv.length + ) { + return undefined; + } + return { cmd, entrypoint, env: publicEnv }; +} + +async function attestVllmContainer(options: { + config: LiveCampaignConfiguration; + manifest: ToolDisclosureManifest; +}): Promise< + Pick< + CampaignAttestation, + "inference_container_id_sha256" | "inference_config_sha256" | "inference_image_digest" + > +> { + const result = await runBoundedCommand({ + command: options.config.docker_bin ?? "docker", + args: ["inspect", options.config.vllm_container_name], + timeoutMs: 30_000, + }); + if (result.exit_code !== 0 || result.timed_out || result.output_truncated) { + throw new Error("vLLM container inspection failed"); + } + const container = selectInspectedContainer( + JSON.parse(result.stdout) as unknown, + options.config.vllm_container_id, + ); + const expected = options.manifest.inference; + const publicConfig = container ? readAttestedVllmConfiguration(container, expected) : undefined; + if (!container || container.Image !== expected.container_digest || !publicConfig) { + throw new Error("live vLLM container does not match the frozen manifest"); + } + return { + inference_container_id_sha256: sha256(container.Id), + inference_config_sha256: sha256(JSON.stringify(publicConfig)), + inference_image_digest: container.Image, + }; +} + +async function attestSandbox(options: { + config: LiveCampaignConfiguration; + manifest: ToolDisclosureManifest; + cell: string; +}): Promise { + const sandboxName = options.config.sandbox_names[options.cell]; + const instanceId = options.config.sandbox_instance_ids[options.cell]; + const imageDigest = options.manifest.environment.sandbox_image_digests?.[options.cell] ?? ""; + if (!instanceId) throw new Error(`live config is missing sandbox instance ID ${options.cell}`); + const result = await runBoundedCommand({ + command: options.config.nemoclaw_bin ?? "nemoclaw", + args: [sandboxName, "status", "--json"], + timeoutMs: 120_000, + }); + if (result.exit_code !== 0 || result.timed_out || result.output_truncated) { + throw new Error(`sandbox status attestation failed for ${options.cell}`); + } + const status = JSON.parse(result.stdout) as { + found?: boolean; + agent?: string; + gatewayState?: string; + }; + const [agent, mode] = options.cell.split(":"); + if (!status.found || status.agent !== agent || status.gatewayState !== "present") { + throw new Error(`sandbox status does not attest the expected cell ${options.cell}`); + } + const containerName = options.config.sandbox_container_names[options.cell]; + if (!containerName) throw new Error(`live config is missing container name ${options.cell}`); + const inspected = await runBoundedCommand({ + command: options.config.docker_bin ?? "docker", + args: ["inspect", containerName], + timeoutMs: 30_000, + }); + if (inspected.exit_code !== 0 || inspected.timed_out || inspected.output_truncated) { + throw new Error(`container inspection failed for ${options.cell}`); + } + const container = selectInspectedContainer<{ + Id?: string; + Image?: string; + Config?: { Env?: string[] }; + }>(JSON.parse(inspected.stdout) as unknown, instanceId); + if ( + !container || + container.Image !== imageDigest || + !container.Config?.Env?.includes(`NEMOCLAW_TOOL_DISCLOSURE=${mode}`) + ) { + throw new Error(`live container does not attest the expected cell ${options.cell}`); + } + return { + cell: options.cell, + instance_id_sha256: sha256(instanceId), + status_sha256: sha256(`${result.stdout}\n${inspected.stdout}`), + image_digest: imageDigest, + }; +} + +function recordCampaignAttestation(outputDir: string, attestation: CampaignAttestation): void { + const file = artifactPath(outputDir, "campaign-attestations.json"); + const existing = fs.existsSync(file) + ? (JSON.parse(fs.readFileSync(file, "utf8")) as CampaignAttestation[]) + : []; + const priorSame = existing.find((item) => item.campaign_id === attestation.campaign_id); + if (priorSame) { + if (priorSame.vllm_process_start_time_seconds !== attestation.vllm_process_start_time_seconds) { + throw new Error("campaign resume attempted against a different vLLM process"); + } + const priorInstances = priorSame.sandbox_cells.map((cell) => cell.instance_id_sha256).sort(); + const currentInstances = attestation.sandbox_cells + .map((cell) => cell.instance_id_sha256) + .sort(); + if (JSON.stringify(priorInstances) !== JSON.stringify(currentInstances)) { + throw new Error("campaign resume attempted against different sandbox instances"); + } + return; + } + for (const prior of existing) { + if (prior.vllm_process_start_time_seconds === attestation.vllm_process_start_time_seconds) { + throw new Error("campaigns must use distinct fresh vLLM processes"); + } + const priorIds = new Set(prior.sandbox_cells.map((cell) => cell.instance_id_sha256)); + if (attestation.sandbox_cells.some((cell) => priorIds.has(cell.instance_id_sha256))) { + throw new Error("campaigns must use distinct fresh sandbox instances"); + } + } + writeJsonArtifact(outputDir, "campaign-attestations.json", [...existing, attestation]); +} + +function sandboxKey(run: ScheduledToolDisclosureRun): string { + return `${run.agent}:${run.mode}:${run.catalog_size}`; +} + +function parseCalls(raw: string): RecordedSyntheticCall[] { + return raw + .split(/\r?\n/u) + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as RecordedSyntheticCall); +} + +function isContextOverflow(stdout: string, stderr: string): boolean { + return /context(?:_| )length(?:_| )exceeded|maximum context length|context window|too many tokens/iu.test( + `${stdout}\n${stderr}`, + ); +} + +export function classifyInvocationFailure(options: { + phase: ScheduledToolDisclosureRun["phase"]; + exitCode: number | null; + timedOut: boolean; + stdout: string; + stderr: string; +}): "context-overflow" | undefined { + return options.phase !== "static-visibility" && + !options.timedOut && + options.exitCode !== null && + options.exitCode !== 0 && + isContextOverflow(options.stdout, options.stderr) + ? "context-overflow" + : undefined; +} + +async function command(built: ReturnType, timeoutMs: number) { + return await runBoundedCommand({ + command: built.command, + args: built.args, + timeoutMs, + redactions: built.redactions, + }); +} + +async function configureMcpSandbox(options: { + config: LiveCampaignConfiguration; + agent: ToolDisclosureAgent; + sandboxName: string; + url: string; + token: string; + tokenEnv: string; +}): Promise { + const binary = options.config.nemoclaw_bin ?? "nemoclaw"; + const invoke = async (args: string[]) => { + const result = await runBoundedCommand({ + command: binary, + args, + env: { ...process.env, [options.tokenEnv]: options.token }, + timeoutMs: 20 * 60_000, + redactions: [options.token], + }); + return result; + }; + const run = async (args: string[]): Promise => { + const result = await invoke(args); + if (result.exit_code !== 0 || result.timed_out || result.output_truncated) { + throw new Error(`MCP setup failed for ${options.agent} sandbox`); + } + }; + if (options.agent === "hermes") { + await run([ + options.sandboxName, + "shields", + "down", + "--timeout", + "15m", + "--reason", + "tool disclosure performance test setup", + ]); + } + try { + const listed = await invoke([options.sandboxName, "mcp", "list", "--json"]); + if (listed.exit_code !== 0 || listed.timed_out || listed.output_truncated) { + throw new Error(`MCP inventory failed for ${options.agent} sandbox`); + } + if (/"(?:name|server)"\s*:\s*"performance-test"/u.test(listed.stdout)) { + await run([options.sandboxName, "mcp", "remove", "performance-test", "--force"]); + } + await run([ + options.sandboxName, + "mcp", + "add", + "performance-test", + "--url", + options.url, + "--env", + options.tokenEnv, + ]); + } finally { + if (options.agent === "hermes") await run([options.sandboxName, "shields", "up"]); + } +} + +function taskMap( + primary: SyntheticTaskSet, + stress: SyntheticTaskSet, +): Map { + return new Map([...primary.tasks, ...stress.tasks].map((task) => [task.id, task] as const)); +} + +/** Execute one frozen campaign against already-created fresh sandboxes and vLLM. */ +export async function executeCampaign(options: { + outputDir: string; + config: LiveCampaignConfiguration; + manifest: ToolDisclosureManifest; + catalog: SyntheticCatalog; + primaryTasks: SyntheticTaskSet; + stressTasks: SyntheticTaskSet; + schedule: readonly ScheduledToolDisclosureRun[]; +}): Promise { + if (options.config.campaign !== 1 && options.config.campaign !== 2) { + throw new Error("live campaign configuration must select campaign 1 or 2"); + } + const scheduled = options.schedule.filter((run) => run.campaign === options.config.campaign); + const expectedKeys = new Set(scheduled.map(sandboxKey)); + for (const key of expectedKeys) { + if (!options.config.sandbox_names[key]) + throw new Error(`live config is missing sandbox ${key}`); + } + const names = [...expectedKeys].map((key) => options.config.sandbox_names[key]); + if (new Set(names).size !== names.length) + throw new Error("every performance test cell needs a unique sandbox"); + const instanceIds = [...expectedKeys].map((key) => options.config.sandbox_instance_ids[key]); + if (instanceIds.some((value) => !value) || new Set(instanceIds).size !== instanceIds.length) { + throw new Error("every performance test cell needs a unique live sandbox instance ID"); + } + const containerNames = [...expectedKeys].map( + (key) => options.config.sandbox_container_names[key], + ); + if ( + containerNames.some((value) => !value) || + new Set(containerNames).size !== containerNames.length + ) { + throw new Error("every performance test cell needs a unique live container name"); + } + if (options.config.tokenizer_model !== options.manifest.inference.model_id) { + throw new Error("tokenizer_model must exactly match manifest.inference.model_id"); + } + if ( + new URL(options.config.upstream_vllm_url).origin !== + new URL(options.config.telemetry_url).origin + ) { + throw new Error("inference and telemetry must address the same vLLM process"); + } + const journal = recoverAttemptJournal(options.outputDir); + const existing = journal.map((entry) => entry.run); + const complete = new Set( + existing.filter((run) => run.outcome !== "setup-error").map((run) => run.run_id), + ); + const token = randomBytes(32).toString("hex"); + const tokenEnv = "TOOL_DISCLOSURE_PERFORMANCE_TEST_MCP_TOKEN"; + const proxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: options.config.upstream_vllm_url, + port: options.config.recorder_port, + requiredTemperature: 0, + }); + const servers = new Map(); + const tunnels = new Map(); + try { + const proxyAddress = await proxy.start(); + if (proxyAddress.base_url !== options.config.managed_inference_base_url) { + throw new Error("managed inference route does not match the started recorder address"); + } + const processStart = await readVllmProcessStartTime(options.config.telemetry_url); + const inferenceAttestation = await attestVllmContainer({ + config: options.config, + manifest: options.manifest, + }); + const sandboxCells: CampaignAttestation["sandbox_cells"][number][] = []; + for (const cell of [...expectedKeys].sort()) { + sandboxCells.push( + await attestSandbox({ config: options.config, manifest: options.manifest, cell }), + ); + } + recordCampaignAttestation(options.outputDir, { + campaign_id: `campaign-${options.config.campaign}`, + vllm_process_start_time_seconds: processStart, + ...inferenceAttestation, + sandbox_cells: sandboxCells, + }); + for (const size of [16, 64, 256, 512, 2_209]) { + const server = new SyntheticMcpServer(generateCatalogPrefix(options.catalog, size), token); + const address = await server.start(); + servers.set(size, server); + tunnels.set( + size, + await startQuickTunnel({ + port: address.port, + binary: options.config.cloudflared_bin, + }), + ); + } + for (const key of expectedKeys) { + const [agent, , sizeText] = key.split(":") as [ToolDisclosureAgent, string, string]; + if (agent === "openclaw") continue; + await configureMcpSandbox({ + config: options.config, + agent, + sandboxName: options.config.sandbox_names[key], + url: tunnels.get(Number(sizeText))?.mcpUrl ?? "", + token, + tokenEnv, + }); + } + + const tasks = taskMap(options.primaryTasks, options.stressTasks); + const timeoutMs = options.config.timeout_ms ?? 10 * 60_000; + for (const run of scheduled) { + if (complete.has(run.run_id)) continue; + const task = run.task_id ? tasks.get(run.task_id) : undefined; + const sandboxName = options.config.sandbox_names[sandboxKey(run)]; + const mcp = run.agent === "openclaw" ? undefined : servers.get(run.catalog_size); + const firstAttempt = nextSetupAttempt( + journal, + run.run_id, + options.manifest.protocol.retry_setup_failures, + ); + for ( + let attempt = firstAttempt; + attempt <= options.manifest.protocol.retry_setup_failures; + attempt += 1 + ) { + let invocationAttempted = false; + try { + if (run.agent === "openclaw") { + const reset = await command( + buildOpenClawCallLogCommand({ + openshellBin: options.config.openshell_bin, + sandboxName, + action: "reset", + }), + timeoutMs, + ); + if (reset.exit_code !== 0 || reset.timed_out) + throw new Error("OpenClaw call-log reset failed"); + } + mcp?.beginRun(run.run_id); + proxy.beginRun(run.run_id); + const before = await readVllmTokenSnapshot(options.config.telemetry_url); + const driverCommand = buildAgentDriverCommand({ + openshellBin: options.config.openshell_bin, + sandboxName, + agent: run.agent, + prompt: task?.prompt ?? "Without using tools, reply exactly STATIC-CAPTURE.", + sessionId: run.run_id, + }); + invocationAttempted = true; + const invocation = await command(driverCommand, timeoutMs); + if (invocation.output_truncated) + throw new Error("agent output exceeded the evidence bound"); + const after = await readVllmTokenSnapshot(options.config.telemetry_url); + const recorderEvents = proxy.endRun(); + const snapshots = proxy.consumeToolSchemaSnapshots(run.run_id); + if (!snapshots[0]) throw new Error("recorder did not capture an initial tool schema"); + const initialSchemaTokens = await countTokensWithVllm( + options.config.telemetry_url, + options.config.tokenizer_model, + snapshots[0].canonical_tools_json, + ); + let calls: RecordedSyntheticCall[]; + if (run.agent === "openclaw") { + const callLog = await command( + buildOpenClawCallLogCommand({ + openshellBin: options.config.openshell_bin, + sandboxName, + action: "read", + }), + timeoutMs, + ); + if (callLog.exit_code !== 0 || callLog.timed_out || callLog.output_truncated) { + throw new Error("OpenClaw call-log read failed"); + } + calls = parseCalls(callLog.stdout); + } else { + calls = mcp?.endRun() ?? []; + } + const tokens = tokenDelta(before, after); + if (!tokens.available) throw new Error("vLLM token counters were unavailable or reset"); + const finalOutput = extractFinalAssistantOutput(run.agent, invocation.stdout); + const classifiedFailure = classifyInvocationFailure({ + phase: run.phase, + exitCode: invocation.exit_code, + timedOut: invocation.timed_out, + stdout: invocation.stdout, + stderr: invocation.stderr, + }); + const rawEvidence: SanitizedRunEvidence = { + run_id: run.run_id, + recorder_events: recorderEvents, + calls, + invocation: { + exit_code: invocation.exit_code, + timed_out: invocation.timed_out, + elapsed_ms: invocation.elapsed_ms, + }, + initial_schema_tokens: initialSchemaTokens, + ...(tokens.available + ? { prompt_tokens: tokens.prompt_tokens, completion_tokens: tokens.generation_tokens } + : {}), + final_oracles_present: + !task || task.expected_final_includes.every((oracle) => finalOutput.includes(oracle)), + ...(classifiedFailure ? { failure_outcome: classifiedFailure } : {}), + }; + const record = assembleToolDisclosureRun({ + manifest: options.manifest, + scheduled: run, + task, + calls, + recorderEvents, + invocation: { + exit_code: invocation.exit_code, + timed_out: invocation.timed_out, + elapsed_ms: invocation.elapsed_ms, + final_output: finalOutput, + }, + initialSchemaTokens, + ...(tokens.available + ? { promptTokens: tokens.prompt_tokens, completionTokens: tokens.generation_tokens } + : {}), + ...(classifiedFailure ? { failureOutcome: classifiedFailure } : {}), + }); + const entry = { raw: rawEvidence, run: record } satisfies AttemptJournalEntry; + appendJsonLine(options.outputDir, "attempt-journal.jsonl", entry); + journal.push(entry); + break; + } catch (error) { + try { + proxy.endRun(); + } catch {} + try { + mcp?.endRun(); + } catch {} + assertSetupRetryAllowed(invocationAttempted, run.run_id, error); + const setupEvidence: SanitizedRunEvidence = { + run_id: run.run_id, + recorder_events: [], + calls: [], + invocation: { exit_code: null, timed_out: false, elapsed_ms: 0 }, + initial_schema_tokens: 0, + final_oracles_present: false, + failure_outcome: "setup-error", + }; + const setupRun = assembleToolDisclosureRun({ + manifest: options.manifest, + scheduled: run, + task, + calls: [], + recorderEvents: [], + invocation: { exit_code: null, timed_out: false, elapsed_ms: 0, final_output: "" }, + initialSchemaTokens: 0, + failureOutcome: "setup-error", + }); + const entry = { raw: setupEvidence, run: setupRun } satisfies AttemptJournalEntry; + appendJsonLine(options.outputDir, "attempt-journal.jsonl", entry); + journal.push(entry); + if (attempt === options.manifest.protocol.retry_setup_failures) + throw new Error(`run ${run.run_id} exhausted setup retries`); + } + } + } + materializeAttemptJournal(options.outputDir, journal); + } finally { + await Promise.allSettled([...tunnels.values()].map((tunnel) => tunnel.close())); + await Promise.allSettled([...servers.values()].map((server) => server.stop())); + await proxy.stop(); + } +} diff --git a/scripts/performance/tool-disclosure/grading.ts b/scripts/performance/tool-disclosure/grading.ts new file mode 100644 index 0000000000..7a06a990a2 --- /dev/null +++ b/scripts/performance/tool-disclosure/grading.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { canonicalJson, sha256Hex } from "./catalog"; +import type { SyntheticPerformanceTask } from "./tasks"; +import type { PerformanceTestRunOutcome, RunCorrectness } from "./types"; + +export interface RecordedSyntheticCall { + tool_name: string; + arguments_sha256: string; + result_nonce: string | null; + success: boolean; +} + +export interface GradedTaskRun { + outcome: Extract; + correctness: RunCorrectness; +} + +/** Grade a run only from deterministic call metadata and the final model output. */ +export function gradeTaskRun( + task: SyntheticPerformanceTask, + calls: readonly RecordedSyntheticCall[], + finalOutput: string, +): GradedTaskRun { + const expectedNames = task.expected_calls.map((call) => call.tool_name); + const actualNames = calls.map((call) => call.tool_name); + const expectedHashes = task.expected_calls.map((call) => + sha256Hex(canonicalJson(call.arguments)), + ); + const expectedNonces = task.expected_calls.map((call) => call.result_nonce); + const expectedCallCount = calls.length === task.expected_calls.length; + const expectedToolOrder = + expectedCallCount && actualNames.every((name, index) => name === expectedNames[index]); + const expectedArguments = + expectedCallCount && + calls.every( + (call, index) => + call.success && + call.arguments_sha256 === expectedHashes[index] && + call.result_nonce === expectedNonces[index], + ); + const expectedToolNames = + expectedNames.length === actualNames.length && + [...expectedNames].sort().every((name, index) => name === [...actualNames].sort()[index]); + const oraclePresent = + task.expected_final_includes.length > 0 && + task.expected_final_includes.every((oracle) => finalOutput.includes(oracle)); + const unnecessaryToolCalls = calls.filter( + (call, index) => call.tool_name !== expectedNames[index], + ).length; + const taskSuccess = + expectedToolNames && + expectedToolOrder && + expectedArguments && + expectedCallCount && + oraclePresent && + unnecessaryToolCalls === 0; + + return { + outcome: taskSuccess ? "success" : "incorrect", + correctness: { + task_success: taskSuccess, + expected_tool_names: expectedToolNames, + expected_tool_order: expectedToolOrder, + expected_arguments: expectedArguments, + expected_call_count: expectedCallCount, + nonce_present: oraclePresent, + unnecessary_tool_calls: unnecessaryToolCalls, + }, + }; +} diff --git a/scripts/performance/tool-disclosure/mcp-server.ts b/scripts/performance/tool-disclosure/mcp-server.ts new file mode 100644 index 0000000000..6519377140 --- /dev/null +++ b/scripts/performance/tool-disclosure/mcp-server.ts @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import http, { type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { + canonicalJson, + executeSyntheticTool, + type JsonObject, + type SyntheticCatalog, + type SyntheticTool, +} from "./catalog"; + +const MAX_BODY_BYTES = 8 * 1024 * 1024; +const SAFE_RUN_ID = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,255}$/u; +const NOTIFICATIONS = new Set([ + "notifications/initialized", + "notifications/cancelled", + "notifications/progress", + "notifications/roots/list_changed", + "notifications/elicitation/complete", +]); + +export interface SyntheticMcpCallEvent { + run_id: string; + sequence: number; + tool_name: string; + arguments_sha256: string; + result_nonce: string | null; + success: boolean; +} + +export interface SyntheticMcpServerAddress { + host: "127.0.0.1"; + port: number; + local_url: string; +} + +function jsonResponse(response: ServerResponse, status: number, value: unknown): void { + const body = JSON.stringify(value); + response.writeHead(status, { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + "cache-control": "no-store", + }); + response.end(body); +} + +async function readBody(request: IncomingMessage): Promise { + return await new Promise((resolve) => { + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + const finish = (value: Buffer | null): void => { + if (settled) return; + settled = true; + resolve(value); + }; + request.on("data", (chunk: Buffer | string) => { + if (settled) return; + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += buffer.length; + if (size > MAX_BODY_BYTES) { + request.resume(); + finish(null); + } else { + chunks.push(buffer); + } + }); + request.once("end", () => finish(Buffer.concat(chunks, size))); + request.once("aborted", () => finish(null)); + request.once("error", () => finish(null)); + }); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function asArguments(value: unknown): JsonObject | null { + if (!isRecord(value)) return null; + return value as JsonObject; +} + +function mcpTool(tool: SyntheticTool): Record { + return { + name: tool.definition.function.name, + description: tool.definition.function.description, + inputSchema: tool.definition.function.parameters, + }; +} + +export class SyntheticMcpServer { + private readonly toolsByName: Map; + private readonly bearerToken: string; + private readonly server: http.Server; + private activeRun: { id: string; sequence: number; eventStartIndex: number } | null = null; + private readonly events: SyntheticMcpCallEvent[] = []; + + constructor(catalog: SyntheticCatalog, bearerToken: string) { + if (!bearerToken || bearerToken.length > 4_096) { + throw new Error("performance test MCP bearer token must be non-empty and bounded"); + } + this.bearerToken = bearerToken; + this.toolsByName = new Map( + catalog.tools.map((tool) => [tool.definition.function.name, tool] as const), + ); + if (this.toolsByName.size !== catalog.size) { + throw new Error("performance test MCP catalog contains duplicate tool names"); + } + this.server = http.createServer((request, response) => { + void this.handle(request, response).catch(() => { + jsonResponse(response, 500, { error: "performance test MCP server failure" }); + }); + }); + this.server.requestTimeout = 120_000; + this.server.headersTimeout = 60_000; + } + + async start(port = 0): Promise { + if (!Number.isInteger(port) || port < 0 || port > 65_535) { + throw new Error("performance test MCP port must be between 0 and 65535"); + } + await new Promise((resolve, reject) => { + this.server.once("error", reject); + this.server.listen(port, "127.0.0.1", () => resolve()); + }); + const address = this.server.address() as AddressInfo | null; + if (!address || address.address !== "127.0.0.1") { + await this.stop(); + throw new Error("performance test MCP server failed to bind loopback"); + } + return { + host: "127.0.0.1", + port: address.port, + local_url: `http://127.0.0.1:${address.port}/mcp`, + }; + } + + async stop(): Promise { + await new Promise((resolve) => this.server.close(() => resolve())); + } + + beginRun(runId: string): void { + if (!SAFE_RUN_ID.test(runId)) throw new Error("invalid performance test MCP run id"); + if (this.activeRun) throw new Error("a performance test MCP run is already active"); + this.activeRun = { id: runId, sequence: 0, eventStartIndex: this.events.length }; + } + + endRun(): SyntheticMcpCallEvent[] { + const run = this.activeRun; + if (!run) throw new Error("no performance test MCP run is active"); + this.activeRun = null; + return this.events + .slice(run.eventStartIndex) + .filter((event) => event.run_id === run.id) + .map((event) => ({ ...event })); + } + + private record( + toolName: string, + argumentsValue: JsonObject, + nonce: string | null, + success: boolean, + ): void { + const run = this.activeRun; + if (!run) return; + run.sequence += 1; + this.events.push({ + run_id: run.id, + sequence: run.sequence, + tool_name: toolName, + arguments_sha256: executeArgumentHash(argumentsValue), + result_nonce: nonce, + success, + }); + } + + private async handle(request: IncomingMessage, response: ServerResponse): Promise { + const pathname = new URL(request.url ?? "/", "http://127.0.0.1").pathname; + if (pathname !== "/mcp") { + jsonResponse(response, 404, { error: "not found" }); + return; + } + if (request.method === "HEAD" || request.method === "GET") { + response.writeHead(405, { allow: "POST" }); + response.end(); + return; + } + if (request.method !== "POST") { + jsonResponse(response, 405, { error: "method not allowed" }); + return; + } + if (request.headers.authorization !== `Bearer ${this.bearerToken}`) { + jsonResponse(response, 401, { error: "authentication required" }); + return; + } + const body = await readBody(request); + if (!body) { + jsonResponse(response, 413, { error: "request rejected" }); + return; + } + let payload: unknown; + try { + payload = JSON.parse(body.toString("utf8")); + } catch { + jsonResponse(response, 400, { error: "invalid JSON" }); + return; + } + if (!isRecord(payload) || typeof payload.method !== "string") { + jsonResponse(response, 400, { error: "invalid JSON-RPC request" }); + return; + } + if (NOTIFICATIONS.has(payload.method)) { + response.writeHead(202); + response.end(); + return; + } + const id = payload.id ?? 1; + if (payload.method === "initialize") { + const params = isRecord(payload.params) ? payload.params : {}; + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + result: { + protocolVersion: + typeof params.protocolVersion === "string" ? params.protocolVersion : "2025-03-26", + capabilities: { tools: {} }, + serverInfo: { name: "nemoclaw-tool-disclosure-performance-test", version: "1.0.0" }, + }, + }); + return; + } + if (payload.method === "tools/list") { + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + result: { tools: [...this.toolsByName.values()].map(mcpTool) }, + }); + return; + } + if (payload.method === "tools/call") { + const params = isRecord(payload.params) ? payload.params : {}; + const name = typeof params.name === "string" ? params.name : ""; + const argumentsValue = asArguments(params.arguments); + const tool = this.toolsByName.get(name); + if (!tool || !argumentsValue) { + this.record(name || "[invalid-name]", argumentsValue ?? {}, null, false); + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + error: { code: -32602, message: "invalid performance test tool call" }, + }); + return; + } + try { + const result = executeSyntheticTool(tool, argumentsValue); + this.record(name, argumentsValue, result.nonce, true); + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + result: { content: [{ type: "text", text: JSON.stringify(result) }], isError: false }, + }); + } catch { + this.record(name, argumentsValue, null, false); + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + error: { code: -32602, message: "invalid performance test tool arguments" }, + }); + } + return; + } + jsonResponse(response, 200, { + jsonrpc: "2.0", + id, + error: { code: -32601, message: "method not found" }, + }); + } +} + +export function executeArgumentHash(argumentsValue: JsonObject): string { + return createHash("sha256").update(canonicalJson(argumentsValue), "utf8").digest("hex"); +} diff --git a/scripts/performance/tool-disclosure/openclaw-fixture.ts b/scripts/performance/tool-disclosure/openclaw-fixture.ts new file mode 100644 index 0000000000..07ac22d0e6 --- /dev/null +++ b/scripts/performance/tool-disclosure/openclaw-fixture.ts @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import type { SyntheticCatalog } from "./catalog"; + +const SAFE_FIXTURE_DIR = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u; +const SAFE_IMMUTABLE_SANDBOX_BASE = + /^[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]{1,5})?(?:\/[a-z0-9]+(?:[._-][a-z0-9]+)*)+@sha256:[a-f0-9]{64}$/u; +const WHITESPACE_OR_CONTROL = /[\s\u0000-\u001f\u007f]/u; + +export interface OpenClawFixtureOptions { + outputDir: string; + catalog: SyntheticCatalog; + sandboxBase: string; +} + +export interface OpenClawFixtureResult { + directory: string; + files: readonly string[]; +} + +function writeNewFile(directory: string, name: string, content: string): void { + const destination = path.join(directory, name); + if (path.dirname(destination) !== directory) + throw new Error("fixture file escaped output directory"); + fs.writeFileSync(destination, content, { encoding: "utf8", flag: "wx", mode: 0o600 }); +} + +/** Reject mutable or shell/Dockerfile-active image references before writing a fixture. */ +export function assertImmutableSandboxBase(image: string): void { + if (WHITESPACE_OR_CONTROL.test(image) || !SAFE_IMMUTABLE_SANDBOX_BASE.test(image)) { + throw new Error( + "sandbox base must be a canonical registry/repository reference with an immutable sha256 digest", + ); + } +} + +function pluginSource(): string { + return `// Generated by the NemoClaw progressive tool-disclosure performance test.\n\ +import { createHash } from "node:crypto";\n\ +import fs from "node:fs";\n\ +\n\ +const catalog = JSON.parse(fs.readFileSync(new URL("./catalog.json", import.meta.url), "utf8"));\n\ +const callsPath = process.env.NEMOCLAW_PERFORMANCE_TEST_CALLS_PATH || "/sandbox/.nemoclaw-performance-test/calls.jsonl";\n\ +\n\ +function canonical(value) {\n\ + if (Array.isArray(value)) return "[" + value.map(canonical).join(",") + "]";\n\ + if (value && typeof value === "object") {\n\ + return "{" + Object.keys(value).sort().map((key) => JSON.stringify(key) + ":" + canonical(value[key])).join(",") + "}";\n\ + }\n\ + return JSON.stringify(value);\n\ +}\n\ +\n\ +function sha256(value) {\n\ + return createHash("sha256").update(value, "utf8").digest("hex");\n\ +}\n\ +\n\ +function execute(tool, args) {\n\ + const argumentsSha256 = sha256(canonical(args));\n\ + const nonce = "nonce_" + sha256(tool.handler.key + "\\0" + canonical(args)).slice(0, 20);\n\ + const result = {\n\ + ok: true,\n\ + tool_name: tool.definition.function.name,\n\ + category: tool.category,\n\ + operation: tool.operation,\n\ + nonce,\n\ + arguments_sha256: argumentsSha256,\n\ + summary: tool.operation.replaceAll("_", " ") + " completed for " + String(args.resource_id),\n\ + };\n\ + fs.appendFileSync(callsPath, JSON.stringify({\n\ + tool_name: result.tool_name,\n\ + arguments_sha256: argumentsSha256,\n\ + result_nonce: nonce,\n\ + success: true,\n\ + }) + "\\n", { encoding: "utf8", mode: 0o660 });\n\ + return result;\n\ +}\n\ +\n\ +export default function registerPerformanceTestTools(api) {\n\ + if (api.registrationMode && api.registrationMode !== "full") return;\n\ + for (const tool of catalog.tools) {\n\ + const definition = tool.definition.function;\n\ + api.registerTool({\n\ + name: definition.name,\n\ + label: definition.name,\n\ + description: definition.description,\n\ + parameters: definition.parameters,\n\ + async execute(_toolCallId, params) {\n\ + const result = execute(tool, params);\n\ + return {\n\ + content: [{ type: "text", text: JSON.stringify(result) }],\n\ + details: result,\n\ + };\n\ + },\n\ + });\n\ + }\n\ +}\n`; +} + +function dockerfile(sandboxBase: string): string { + return `ARG SANDBOX_BASE=${sandboxBase}\n\ +FROM \${SANDBOX_BASE}\n\ +\n\ +ARG NEMOCLAW_TOOL_DISCLOSURE=progressive\n\ +ENV NEMOCLAW_TOOL_DISCLOSURE=\${NEMOCLAW_TOOL_DISCLOSURE}\n\ +\n\ +COPY plugin/ /opt/nemoclaw-tool-disclosure-performance-test/\n\ +RUN mkdir -p /sandbox/.openclaw/extensions \\\n && cp -a /opt/nemoclaw-tool-disclosure-performance-test /sandbox/.openclaw/extensions/nemoclaw-tool-disclosure-performance-test \\\n && install -d -m 2770 -o sandbox -g sandbox /sandbox/.nemoclaw-performance-test \\\n && openclaw doctor --fix\n\ +\n\ +WORKDIR /opt/nemoclaw\n`; +} + +/** Guard the renderer boundary so the validated base and emitted ARG cannot diverge. */ +export function assertDockerfileSandboxBase(content: string, expected: string): void { + assertImmutableSandboxBase(expected); + const [argument, from] = content.split("\n", 2); + if (argument !== `ARG SANDBOX_BASE=${expected}` || from !== "FROM ${SANDBOX_BASE}") { + throw new Error("generated Dockerfile sandbox base does not match"); + } +} + +/** Generate a self-contained custom-image context for an OpenClaw catalog size. */ +export function writeOpenClawFixture(options: OpenClawFixtureOptions): OpenClawFixtureResult { + assertImmutableSandboxBase(options.sandboxBase); + const renderedDockerfile = dockerfile(options.sandboxBase); + assertDockerfileSandboxBase(renderedDockerfile, options.sandboxBase); + const root = path.resolve(options.outputDir); + if (fs.existsSync(root)) { + const stat = fs.lstatSync(root); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error("OpenClaw fixture output must be a real directory"); + } + if (fs.readdirSync(root).length > 0) throw new Error("OpenClaw fixture output must be empty"); + } else { + fs.mkdirSync(root, { recursive: true, mode: 0o700 }); + } + if (!SAFE_FIXTURE_DIR.test(path.basename(root))) { + throw new Error("OpenClaw fixture directory has an unsafe name"); + } + + const pluginDir = path.join(root, "plugin"); + fs.mkdirSync(pluginDir, { mode: 0o700 }); + const names = options.catalog.tools.map((tool) => tool.definition.function.name); + writeNewFile(pluginDir, "catalog.json", `${JSON.stringify(options.catalog, null, 2)}\n`); + writeNewFile(pluginDir, "index.js", pluginSource()); + writeNewFile( + pluginDir, + "package.json", + `${JSON.stringify( + { + name: "nemoclaw-tool-disclosure-performance-test", + version: "1.0.0", + private: true, + type: "module", + openclaw: { extensions: ["./index.js"] }, + }, + null, + 2, + )}\n`, + ); + writeNewFile( + pluginDir, + "openclaw.plugin.json", + `${JSON.stringify( + { + id: "nemoclaw-tool-disclosure-performance-test", + name: "NemoClaw Tool Disclosure Performance Test", + version: "1.0.0", + description: "Deterministic synthetic tools for progressive disclosure performance tests.", + contracts: { tools: names }, + configSchema: { type: "object", properties: {}, additionalProperties: false }, + }, + null, + 2, + )}\n`, + ); + writeNewFile(root, "Dockerfile", renderedDockerfile); + writeNewFile(root, ".dockerignore", "*\n!Dockerfile\n!plugin\n!plugin/**\n"); + + return { + directory: root, + files: [ + ".dockerignore", + "Dockerfile", + "plugin/catalog.json", + "plugin/index.js", + "plugin/openclaw.plugin.json", + "plugin/package.json", + ], + }; +} diff --git a/scripts/performance/tool-disclosure/quick-tunnel.ts b/scripts/performance/tool-disclosure/quick-tunnel.ts new file mode 100644 index 0000000000..89e57cbb17 --- /dev/null +++ b/scripts/performance/tool-disclosure/quick-tunnel.ts @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const ORIGIN_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com(?=$|[\s"'\\/])/giu; +// Ambient HOME/XDG_CONFIG_HOME are not allowlisted; the child receives an +// isolated replacement so it cannot load operator credentials or configuration. +const SAFE_ENV = new Set([ + "PATH", + "TMPDIR", + "TMP", + "TEMP", + "LANG", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +]); + +export interface QuickTunnel { + origin: string; + mcpUrl: string; + close(): Promise; +} + +export type QuickTunnelProtocol = "auto" | "http2" | "quic"; + +export function parseQuickTunnelOrigin(text: string): string | null { + return [...text.matchAll(ORIGIN_PATTERN)].at(-1)?.[0] ?? null; +} + +export function buildQuickTunnelArgs( + port: number, + protocol: QuickTunnelProtocol = "http2", +): string[] { + if (!Number.isInteger(port) || port < 1 || port > 65_535) { + throw new Error("quick-tunnel origin port must be between 1 and 65535"); + } + return [ + "tunnel", + "--config=", + "--no-autoupdate", + "--protocol", + protocol, + "--url", + `http://127.0.0.1:${port}`, + "--loglevel", + "info", + ]; +} + +export function buildQuickTunnelEnvironment( + env: NodeJS.ProcessEnv, + isolatedHome: string, +): NodeJS.ProcessEnv { + if (!path.isAbsolute(isolatedHome)) throw new Error("quick-tunnel home must be absolute"); + const selected: NodeJS.ProcessEnv = {}; + for (const [name, value] of Object.entries(env)) { + if (value !== undefined && (SAFE_ENV.has(name) || name.startsWith("LC_"))) { + selected[name] = value; + } + } + return { ...selected, HOME: isolatedHome, XDG_CONFIG_HOME: isolatedHome }; +} + +function stopChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); + return new Promise((resolve) => { + const timer = setTimeout(() => { + try { + child.kill("SIGKILL"); + } catch { + // The process exited between the state check and signal. + } + }, 5_000); + timer.unref(); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + try { + child.kill("SIGTERM"); + } catch { + clearTimeout(timer); + resolve(); + } + }); +} + +function readLogTail(file: string, maxBytes = 8_192): string { + let descriptor: number | undefined; + try { + descriptor = fs.openSync(file, "r"); + const size = fs.fstatSync(descriptor).size; + const length = Math.min(size, maxBytes); + const buffer = Buffer.alloc(length); + fs.readSync(descriptor, buffer, 0, length, Math.max(0, size - length)); + return buffer.toString("utf8"); + } catch { + return ""; + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + +async function probe(origin: string, fetchImpl: typeof fetch): Promise { + try { + const response = await fetchImpl(`${origin}/mcp`, { + method: "HEAD", + redirect: "error", + signal: AbortSignal.timeout(5_000), + }); + await response.body?.cancel(); + return response.status; + } catch { + return null; + } +} + +function sanitizedLogDiagnostic(file: string): string { + return readLogTail(file, 2_048) + .replace(ORIGIN_PATTERN, "https://redacted.trycloudflare.com") + .replace(/\s+/gu, " ") + .trim() + .slice(-1_500); +} + +export async function startQuickTunnel(options: { + port: number; + binary?: string; + timeoutMs?: number; + fetchImpl?: typeof fetch; + env?: NodeJS.ProcessEnv; + protocol?: QuickTunnelProtocol; +}): Promise { + const args = buildQuickTunnelArgs(options.port, options.protocol); + const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-")); + const logFile = path.join(isolatedHome, "cloudflared.log"); + const removeIsolatedHome = (): void => fs.rmSync(isolatedHome, { recursive: true, force: true }); + let child: ChildProcess; + let logDescriptor: number | undefined; + try { + logDescriptor = fs.openSync(logFile, "a"); + child = spawn(options.binary ?? "cloudflared", args, { + detached: false, + env: buildQuickTunnelEnvironment(options.env ?? process.env, isolatedHome), + stdio: ["ignore", logDescriptor, logDescriptor], + }); + } catch (error) { + if (logDescriptor !== undefined) fs.closeSync(logDescriptor); + removeIsolatedHome(); + throw error; + } + if (logDescriptor !== undefined) fs.closeSync(logDescriptor); + const timeoutMs = options.timeoutMs ?? 45_000; + const fetchImpl = options.fetchImpl ?? fetch; + let origin: string | null = null; + let lastProbeStatus: number | null | undefined; + let spawnError: Error | undefined; + child.once("error", (error) => { + spawnError = error; + }); + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + origin = parseQuickTunnelOrigin(readLogTail(logFile)) ?? origin; + if (spawnError) break; + if (child.exitCode !== null || child.signalCode !== null) break; + if (origin) lastProbeStatus = await probe(origin, fetchImpl); + if (origin && lastProbeStatus === 405) { + const published = origin; + return { + origin: published, + mcpUrl: `${published}/mcp`, + close: async () => { + try { + await stopChild(child); + } finally { + removeIsolatedHome(); + } + }, + }; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + const diagnostic = sanitizedLogDiagnostic(logFile); + const childState = + child.exitCode !== null + ? `exit-${child.exitCode}` + : child.signalCode !== null + ? `signal-${child.signalCode}` + : "running"; + try { + await stopChild(child); + } finally { + removeIsolatedHome(); + } + if (spawnError) throw new Error(`cloudflared quick tunnel failed to start (${spawnError.name})`); + throw new Error( + [ + "cloudflared quick tunnel did not become ready before the timeout", + `origin_published=${origin !== null}`, + `last_probe_status=${lastProbeStatus === undefined ? "not-attempted" : (lastProbeStatus ?? "fetch-error")}`, + `child_state=${childState}`, + diagnostic ? `log=${diagnostic}` : "log=empty", + ].join("; "), + ); +} diff --git a/scripts/performance/tool-disclosure/recorder.ts b/scripts/performance/tool-disclosure/recorder.ts new file mode 100644 index 0000000000..7a793d55a7 --- /dev/null +++ b/scripts/performance/tool-disclosure/recorder.ts @@ -0,0 +1,1018 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; +import http, { + type IncomingHttpHeaders, + type IncomingMessage, + type OutgoingHttpHeaders, + type Server, + type ServerResponse, +} from "node:http"; +import https from "node:https"; +import { isIP, type Socket } from "node:net"; +import { performance } from "node:perf_hooks"; + +import { canonicalJson, type JsonValue, sha256Hex } from "./catalog"; + +const LOOPBACK_HOST = "127.0.0.1"; +const DEFAULT_MAX_REQUEST_BODY_BYTES = 64 * 1024 * 1024; +const DEFAULT_REQUEST_TIMEOUT_MS = 120_000; +const SAFE_RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/; +const SAFE_TOOL_NAME = /^[A-Za-z0-9_-]{1,128}$/; + +const HOP_BY_HOP_HEADERS = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", +]); + +export type RecorderEndpoint = + | "chat-completions" + | "responses" + | "completions" + | "embeddings" + | "tokenize" + | "models" + | "other-v1"; + +export type RecorderMethod = "GET" | "POST" | "OTHER"; + +export type RecorderOutcome = + | "completed" + | "request-rejected" + | "upstream-timeout" + | "upstream-connection-error" + | "upstream-response-error" + | "client-aborted"; + +export type RecorderErrorReason = + | "request-body-too-large" + | "request-timeout" + | "client-request-error" + | "upstream-timeout" + | "upstream-connection-error" + | "upstream-response-error" + | "proxy-failure" + | "client-aborted"; + +/** + * A deliberately content-free record of one proxied /v1 request. + * + * Do not add URLs, headers, request/response bodies, model identifiers, tool + * descriptions, or thrown error messages to this interface. This object is + * suitable for direct inclusion in the public performance test evidence bundle. + */ +export interface ToolDisclosureRecordingEvent { + run_id: string; + request_sequence: number; + model_call_sequence: number | null; + endpoint: RecorderEndpoint; + method: RecorderMethod; + visible_tool_count: number; + canonical_tools_json_bytes: number; + tools_sha256: string | null; + tool_names: readonly string[]; + streaming: boolean | null; + status_code: number | null; + started_monotonic_ms: number; + first_byte_monotonic_ms: number | null; + ended_monotonic_ms: number; + duration_ms: number; + time_to_first_byte_ms: number | null; + outcome: RecorderOutcome; + error_reason: RecorderErrorReason | null; +} + +export interface RecordingProxyOptions { + upstreamBaseUrl: string; + /** Explicitly allow a fixed non-loopback HTTPS upstream without URL-embedded credentials. */ + allowRemoteHttpsUpstream?: boolean; + /** Defaults to loopback. A private IPv4 bridge requires explicit opt-in and auth translation. */ + listenHost?: string; + allowAuthenticatedPrivateIpv4Listener?: boolean; + /** Runtime-only exact bearer header checked before request reading, recording, or transformation. */ + requiredAuthorization?: string; + /** Runtime-only bearer header that replaces the accepted ingress credential upstream. */ + upstreamAuthorization?: string; + /** Zero selects an ephemeral port. */ + port?: number; + maxRequestBodyBytes?: number; + requestTimeoutMs?: number; + requiredTemperature?: number; + requestTransform?: RecordingProxyRequestTransform; +} + +/** + * Private request content supplied only to an opt-in transform invocation. + * + * The body is a defensive copy and is never retained by the proxy. Transform + * implementations must likewise avoid logging or retaining it. Public evidence + * continues to contain only the content-free metrics derived after transformation. + */ +export interface RecordingProxyTransformInput { + readonly runId: string | null; + readonly endpoint: RecorderEndpoint; + readonly method: RecorderMethod; + readonly modelCallSequence: number | null; + readonly body: Buffer; + readonly signal: AbortSignal; +} + +/** Return the complete request body to forward upstream. */ +export type RecordingProxyRequestTransform = ( + input: RecordingProxyTransformInput, +) => Buffer | Promise; + +export interface RecordingProxyAddress { + host: string; + port: number; + base_url: string; +} + +/** + * Ephemeral schema material for exact tokenizer counting. This type must never + * be serialized into the public evidence bundle. + */ +export interface EphemeralToolSchemaSnapshot { + run_id: string; + model_call_sequence: number; + canonical_tools_json: string; +} + +interface ToolMetrics { + visibleToolCount: number; + canonicalToolsJsonBytes: number; + toolsSha256: string | null; + toolNames: readonly string[]; + canonicalToolsJson: string | null; + streaming: boolean | null; + temperature: number | null; +} + +interface PendingRecording { + runId: string; + requestSequence: number; + modelCallSequence: number | null; + endpoint: RecorderEndpoint; + method: RecorderMethod; + startedAt: number; + metrics: ToolMetrics; + finalized: boolean; +} + +interface ActiveRun { + id: string; + requestSequence: number; + modelCallSequence: number; + pendingRequests: number; + eventStartIndex: number; +} + +interface ForwardContext { + bodyComplete: boolean; + transformComplete: boolean; + cancelled: boolean; + timedOut: boolean; + abortController: AbortController; + upstreamRequest: http.ClientRequest | null; +} + +interface FinalizeOptions { + statusCode: number | null; + firstByteAt?: number; + outcome: RecorderOutcome; + errorReason?: RecorderErrorReason; +} + +type BodyReadResult = + | { kind: "body"; body: Buffer } + | { kind: "too-large" } + | { kind: "client-error" }; + +function validatePositiveInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${label} must be a positive integer`); + } +} + +function isPrivateIpv4(value: string): boolean { + if (isIP(value) !== 4) return false; + const [first, second] = value.split(".").map(Number); + return ( + first === 10 || + (first === 172 && second >= 16 && second <= 31) || + (first === 192 && second === 168) + ); +} + +function authorizationHeader(value: string | undefined, label: string): string | undefined { + if (value === undefined) return undefined; + const token = value.startsWith("Bearer ") ? value.slice("Bearer ".length) : ""; + if (!token || token.length > 4_096 || /\s/u.test(token)) { + throw new Error(`${label} must be a bounded Bearer authorization header`); + } + return value; +} + +function authorizationMatches(value: string | undefined, expected: string): boolean { + if (value === undefined) return false; + const actualDigest = createHash("sha256").update(value).digest(); + const expectedDigest = createHash("sha256").update(expected).digest(); + return timingSafeEqual(actualDigest, expectedDigest); +} + +function parseUpstreamBaseUrl(value: string, allowRemoteHttpsUpstream: boolean): URL { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error("upstreamBaseUrl must be a valid HTTP(S) URL"); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("upstreamBaseUrl must use HTTP or HTTPS"); + } + const hostname = url.hostname.replace(/^\[/u, "").replace(/\]$/u, "").toLowerCase(); + // URL canonicalizes equivalent IPv6 spellings to ::1. Keep the plaintext + // allowlist exact; IPv4-mapped and scoped IPv6 forms are intentionally rejected. + const loopback = + hostname === "localhost" || + hostname === "::1" || + (isIP(hostname) === 4 && hostname.startsWith("127.")); + if (!loopback && !allowRemoteHttpsUpstream) { + throw new Error("upstreamBaseUrl is allowed only on loopback"); + } + if (!loopback && url.protocol !== "https:") { + throw new Error("a remote recording upstream must use HTTPS"); + } + if (url.username || url.password) { + throw new Error("upstreamBaseUrl must not contain credentials"); + } + // Query values are never recorded, but accepting them in long-lived + // configuration makes it too easy to embed an API key in a report command. + if (url.search || url.hash) { + throw new Error("upstreamBaseUrl must not contain a query or fragment"); + } + // Do not delegate the accepted localhost alias to ambient DNS resolution. + if (hostname === "localhost") { + url.hostname = LOOPBACK_HOST; + } + return url; +} + +function classifyEndpoint(pathname: string): RecorderEndpoint { + const normalized = pathname.replace(/\/+$/, "") || "/"; + if (normalized === "/v1/chat/completions") return "chat-completions"; + if (normalized === "/v1/responses") return "responses"; + if (normalized === "/v1/completions") return "completions"; + if (normalized === "/v1/embeddings") return "embeddings"; + if (normalized === "/v1/tokenize") return "tokenize"; + if (normalized === "/v1/models") return "models"; + return "other-v1"; +} + +function classifyMethod(method: string | undefined): RecorderMethod { + if (method === "GET") return "GET"; + if (method === "POST") return "POST"; + return "OTHER"; +} + +function isModelCall(endpoint: RecorderEndpoint, method: RecorderMethod): boolean { + return ( + method === "POST" && + ["chat-completions", "responses", "completions", "embeddings"].includes(endpoint) + ); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function safeToolName(tool: unknown): string | null { + if (!isRecord(tool)) return null; + let candidate: unknown; + if (isRecord(tool.function)) candidate = tool.function.name; + else candidate = tool.name ?? tool.type; + if (typeof candidate !== "string") return null; + return SAFE_TOOL_NAME.test(candidate) ? candidate : "[invalid-name]"; +} + +function inspectTools(body: Buffer): ToolMetrics { + const empty: ToolMetrics = { + visibleToolCount: 0, + canonicalToolsJsonBytes: 0, + toolsSha256: null, + toolNames: [], + canonicalToolsJson: null, + streaming: null, + temperature: null, + }; + if (body.length === 0) return empty; + + try { + const payload: unknown = JSON.parse(body.toString("utf8")); + if (!isRecord(payload)) return empty; + empty.temperature = typeof payload.temperature === "number" ? payload.temperature : null; + if (!Array.isArray(payload.tools)) return empty; + const canonicalTools = canonicalJson(payload.tools as JsonValue); + return { + visibleToolCount: payload.tools.length, + canonicalToolsJsonBytes: Buffer.byteLength(canonicalTools), + toolsSha256: sha256Hex(canonicalTools), + toolNames: payload.tools.flatMap((tool) => { + const name = safeToolName(tool); + return name === null ? [] : [name]; + }), + canonicalToolsJson: canonicalTools, + streaming: typeof payload.stream === "boolean" ? payload.stream : null, + temperature: typeof payload.temperature === "number" ? payload.temperature : null, + }; + } catch { + // Invalid JSON and pathological nesting are forwarded unchanged, but no + // body-derived diagnostic is retained in the public event stream. + return empty; + } +} + +function connectionHeaderNames(headers: IncomingHttpHeaders): Set { + const value = headers.connection; + const joined = Array.isArray(value) ? value.join(",") : (value ?? ""); + return new Set( + joined + .split(",") + .map((item) => item.trim().toLowerCase()) + .filter(Boolean), + ); +} + +function forwardedRequestHeaders( + headers: IncomingHttpHeaders, + bodyLength: number, + authorization: string | undefined, +): OutgoingHttpHeaders { + const result: OutgoingHttpHeaders = {}; + const connectionNames = connectionHeaderNames(headers); + for (const [name, value] of Object.entries(headers)) { + const normalized = name.toLowerCase(); + if ( + value === undefined || + normalized === "host" || + normalized === "content-length" || + HOP_BY_HOP_HEADERS.has(normalized) || + connectionNames.has(normalized) + ) { + continue; + } + result[name] = value; + } + result["content-length"] = String(bodyLength); + if (authorization !== undefined) result.authorization = authorization; + return result; +} + +function forwardedResponseHeaders(headers: IncomingHttpHeaders): OutgoingHttpHeaders { + const result: OutgoingHttpHeaders = {}; + const connectionNames = connectionHeaderNames(headers); + for (const [name, value] of Object.entries(headers)) { + const normalized = name.toLowerCase(); + if ( + value === undefined || + HOP_BY_HOP_HEADERS.has(normalized) || + connectionNames.has(normalized) + ) { + continue; + } + result[name] = value; + } + return result; +} + +function fixedResponse(response: ServerResponse, status: number, error: string): void { + if (response.headersSent || response.destroyed) return; + const body = JSON.stringify({ error }); + response.writeHead(status, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + "cache-control": "no-store", + }); + response.end(body); +} + +function readBoundedBody(request: IncomingMessage, maxBytes: number): Promise { + return new Promise((resolve) => { + let settled = false; + let size = 0; + let chunks: Buffer[] = []; + + const finish = (result: BodyReadResult): void => { + if (settled) return; + settled = true; + chunks = []; + resolve(result); + }; + + request.on("data", (chunk: Buffer | string) => { + if (settled) return; + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += buffer.length; + if (size > maxBytes) { + request.resume(); + finish({ kind: "too-large" }); + return; + } + chunks.push(buffer); + }); + request.once("end", () => { + if (!settled) finish({ kind: "body", body: Buffer.concat(chunks, size) }); + }); + request.once("aborted", () => finish({ kind: "client-error" })); + request.once("error", () => finish({ kind: "client-error" })); + }); +} + +/** + * OpenAI-compatible recording proxy for performance test measurements. + * The default listener is loopback-only. An authenticated RFC1918 listener is an explicit, + * caller-owned exception for an inspected private bridge. The proxy never logs and keeps only + * content-free events in memory. + */ +export class ToolDisclosureRecordingProxy { + private readonly upstream: URL; + private readonly listenHost: string; + private readonly configuredPort: number; + private readonly maxRequestBodyBytes: number; + private readonly requestTimeoutMs: number; + private readonly requiredTemperature: number | undefined; + private readonly requestTransform: RecordingProxyRequestTransform | undefined; + private readonly requiredAuthorization: string | undefined; + private readonly upstreamAuthorization: string | undefined; + private readonly clock: () => number; + private readonly sockets = new Set(); + private readonly events: ToolDisclosureRecordingEvent[] = []; + private readonly ephemeralSchemas = new Map(); + private server: Server | null = null; + private activeRun: ActiveRun | null = null; + + constructor(options: RecordingProxyOptions) { + const listenHost = options.listenHost ?? LOOPBACK_HOST; + const requiredAuthorization = authorizationHeader( + options.requiredAuthorization, + "requiredAuthorization", + ); + const upstreamAuthorization = authorizationHeader( + options.upstreamAuthorization, + "upstreamAuthorization", + ); + if ((requiredAuthorization === undefined) !== (upstreamAuthorization === undefined)) { + throw new Error("recording proxy authorization translation requires both headers"); + } + if ( + listenHost !== LOOPBACK_HOST && + !( + options.allowAuthenticatedPrivateIpv4Listener === true && + isPrivateIpv4(listenHost) && + requiredAuthorization !== undefined + ) + ) { + throw new Error( + `recording proxy listenHost must be exactly ${LOOPBACK_HOST} unless an authenticated private IPv4 listener is explicitly enabled`, + ); + } + const port = options.port ?? 0; + if (!Number.isSafeInteger(port) || port < 0 || port > 65_535) { + throw new Error("recording proxy port must be an integer from 0 through 65535"); + } + const maxRequestBodyBytes = options.maxRequestBodyBytes ?? DEFAULT_MAX_REQUEST_BODY_BYTES; + const requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + validatePositiveInteger(maxRequestBodyBytes, "maxRequestBodyBytes"); + validatePositiveInteger(requestTimeoutMs, "requestTimeoutMs"); + + this.upstream = parseUpstreamBaseUrl( + options.upstreamBaseUrl, + options.allowRemoteHttpsUpstream === true, + ); + this.listenHost = listenHost; + this.configuredPort = port; + this.maxRequestBodyBytes = maxRequestBodyBytes; + this.requestTimeoutMs = requestTimeoutMs; + this.requiredTemperature = options.requiredTemperature; + this.requestTransform = options.requestTransform; + this.requiredAuthorization = requiredAuthorization; + this.upstreamAuthorization = upstreamAuthorization; + this.clock = () => performance.now(); + } + + async start(): Promise { + if (this.server) throw new Error("recording proxy is already started"); + + const server = http.createServer((request, response) => { + void this.handleRequest(request, response).catch(() => { + // Keep thrown implementation errors content-free at the network and + // evidence boundaries. The performance test runner receives a fixed class. + fixedResponse(response, 502, "recording proxy failure"); + }); + }); + server.requestTimeout = this.requestTimeoutMs; + server.headersTimeout = Math.min(this.requestTimeoutMs, 60_000); + server.maxHeadersCount = 200; + server.on("connection", (socket) => { + this.sockets.add(socket); + socket.once("close", () => this.sockets.delete(socket)); + }); + + await new Promise((resolve, reject) => { + const onError = (error: Error): void => { + server.off("listening", onListening); + reject(error); + }; + const onListening = (): void => { + server.off("error", onError); + resolve(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen(this.configuredPort, this.listenHost); + }); + this.server = server; + + const address = server.address(); + if (!address || typeof address === "string" || address.address !== this.listenHost) { + await this.stop(); + throw new Error("recording proxy failed to bind the required listener address"); + } + return { + host: this.listenHost, + port: address.port, + base_url: `http://${this.listenHost}:${address.port}`, + }; + } + + async stop(): Promise { + const server = this.server; + if (!server) return; + this.server = null; + for (const socket of this.sockets) socket.destroy(); + this.sockets.clear(); + await new Promise((resolve) => server.close(() => resolve())); + } + + beginRun(runId: string = randomUUID()): string { + if (this.activeRun) throw new Error("a recording run is already active"); + if (!SAFE_RUN_ID.test(runId)) { + throw new Error( + "recording run ID must be 1-256 public-safe letters, digits, dots, underscores, colons, or hyphens", + ); + } + this.activeRun = { + id: runId, + requestSequence: 0, + modelCallSequence: 0, + pendingRequests: 0, + eventStartIndex: this.events.length, + }; + return runId; + } + + endRun(): readonly ToolDisclosureRecordingEvent[] { + const run = this.activeRun; + if (!run) throw new Error("no recording run is active"); + if (run.pendingRequests !== 0) { + throw new Error("cannot end a recording run while requests are in flight"); + } + this.activeRun = null; + return this.events + .slice(run.eventStartIndex) + .filter((event) => event.run_id === run.id) + .map((event) => ({ ...event, tool_names: [...event.tool_names] })); + } + + getEvents(runId?: string): readonly ToolDisclosureRecordingEvent[] { + return this.events + .filter((event) => runId === undefined || event.run_id === runId) + .map((event) => ({ ...event, tool_names: [...event.tool_names] })); + } + + resetEvents(): void { + if (this.activeRun) throw new Error("cannot reset events while a recording run is active"); + this.events.length = 0; + this.ephemeralSchemas.clear(); + } + + /** Return and erase private schema content after a run so it can be tokenized exactly. */ + consumeToolSchemaSnapshots(runId: string): readonly EphemeralToolSchemaSnapshot[] { + if (this.activeRun?.id === runId) + throw new Error("cannot consume schemas while a run is active"); + const snapshots = [...this.ephemeralSchemas.values()] + .filter((snapshot) => snapshot.run_id === runId) + .sort((left, right) => left.model_call_sequence - right.model_call_sequence) + .map((snapshot) => ({ ...snapshot })); + for (const snapshot of snapshots) { + this.ephemeralSchemas.delete(`${snapshot.run_id}:${snapshot.model_call_sequence}`); + } + return snapshots; + } + + private reserveRecording( + endpoint: RecorderEndpoint, + method: RecorderMethod, + ): PendingRecording | null { + const run = this.activeRun; + if (!run) return null; + run.requestSequence += 1; + run.pendingRequests += 1; + let modelCallSequence: number | null = null; + if (isModelCall(endpoint, method)) { + run.modelCallSequence += 1; + modelCallSequence = run.modelCallSequence; + } + return { + runId: run.id, + requestSequence: run.requestSequence, + modelCallSequence, + endpoint, + method, + startedAt: this.clock(), + metrics: { + visibleToolCount: 0, + canonicalToolsJsonBytes: 0, + toolsSha256: null, + toolNames: [], + canonicalToolsJson: null, + streaming: null, + temperature: null, + }, + finalized: false, + }; + } + + private finalizeRecording(pending: PendingRecording | null, options: FinalizeOptions): void { + if (!pending || pending.finalized) return; + pending.finalized = true; + const endedAt = this.clock(); + const firstByteAt = options.firstByteAt ?? null; + const event: ToolDisclosureRecordingEvent = { + run_id: pending.runId, + request_sequence: pending.requestSequence, + model_call_sequence: pending.modelCallSequence, + endpoint: pending.endpoint, + method: pending.method, + visible_tool_count: pending.metrics.visibleToolCount, + canonical_tools_json_bytes: pending.metrics.canonicalToolsJsonBytes, + tools_sha256: pending.metrics.toolsSha256, + tool_names: [...pending.metrics.toolNames], + streaming: pending.metrics.streaming, + status_code: options.statusCode, + started_monotonic_ms: pending.startedAt, + first_byte_monotonic_ms: firstByteAt, + ended_monotonic_ms: endedAt, + duration_ms: endedAt - pending.startedAt, + time_to_first_byte_ms: firstByteAt === null ? null : firstByteAt - pending.startedAt, + outcome: options.outcome, + error_reason: options.errorReason ?? null, + }; + this.events.push(Object.freeze(event)); + if (pending.modelCallSequence !== null && pending.metrics.canonicalToolsJson !== null) { + const snapshot: EphemeralToolSchemaSnapshot = { + run_id: pending.runId, + model_call_sequence: pending.modelCallSequence, + canonical_tools_json: pending.metrics.canonicalToolsJson, + }; + this.ephemeralSchemas.set( + `${snapshot.run_id}:${snapshot.model_call_sequence}`, + Object.freeze(snapshot), + ); + } + if (this.activeRun?.id === pending.runId) { + this.activeRun.pendingRequests -= 1; + } + } + + private buildUpstreamUrl(incoming: URL): URL { + const target = new URL(this.upstream.toString()); + const basePath = target.pathname.replace(/\/+$/, ""); + const prefix = basePath.endsWith("/v1") ? basePath.slice(0, -3) : basePath; + target.pathname = `${prefix}${incoming.pathname}` || "/"; + target.search = incoming.search; + return target; + } + + private async transformRequestBody( + body: Buffer, + pending: PendingRecording | null, + endpoint: RecorderEndpoint, + method: RecorderMethod, + signal: AbortSignal, + ): Promise { + if (!this.requestTransform) return body; + const transformed = await this.requestTransform({ + runId: pending?.runId ?? null, + endpoint, + method, + modelCallSequence: pending?.modelCallSequence ?? null, + body: Buffer.from(body), + signal, + }); + if (!Buffer.isBuffer(transformed)) { + throw new TypeError("recording proxy request transform must return a Buffer"); + } + // Detach forwarding from any reference retained by the transform so later + // mutation cannot alter the bytes inspected, recorded, or sent upstream. + return Buffer.from(transformed); + } + + private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise { + let incoming: URL; + try { + incoming = new URL(request.url ?? "/", "http://127.0.0.1"); + } catch { + fixedResponse(response, 400, "invalid request target"); + return; + } + if (incoming.pathname !== "/v1" && !incoming.pathname.startsWith("/v1/")) { + fixedResponse(response, 404, "not found"); + return; + } + if ( + this.requiredAuthorization !== undefined && + !authorizationMatches(request.headers.authorization, this.requiredAuthorization) + ) { + request.resume(); + fixedResponse(response, 401, "unauthorized"); + return; + } + + const endpoint = classifyEndpoint(incoming.pathname); + const method = classifyMethod(request.method); + const pending = this.reserveRecording(endpoint, method); + const context: ForwardContext = { + bodyComplete: false, + transformComplete: false, + cancelled: false, + timedOut: false, + abortController: new AbortController(), + upstreamRequest: null, + }; + request.once("aborted", () => context.abortController.abort()); + response.once("close", () => { + if (!response.writableEnded) context.abortController.abort(); + }); + + const declaredLength = Number(request.headers["content-length"]); + if (Number.isFinite(declaredLength) && declaredLength > this.maxRequestBodyBytes) { + request.resume(); + fixedResponse(response, 413, "request body too large"); + this.finalizeRecording(pending, { + statusCode: 413, + outcome: "request-rejected", + errorReason: "request-body-too-large", + }); + return; + } + + const timer = setTimeout(() => { + context.cancelled = true; + context.timedOut = true; + context.abortController.abort(); + if (context.upstreamRequest) { + context.upstreamRequest.destroy(); + return; + } + const waitingForUpstream = context.bodyComplete && context.transformComplete; + const statusCode = waitingForUpstream ? 504 : 408; + fixedResponse( + response, + statusCode, + waitingForUpstream ? "upstream timeout" : "request timeout", + ); + this.finalizeRecording(pending, { + statusCode, + outcome: waitingForUpstream ? "upstream-timeout" : "request-rejected", + errorReason: waitingForUpstream ? "upstream-timeout" : "request-timeout", + }); + request.destroy(); + }, this.requestTimeoutMs); + timer.unref(); + + try { + const bodyResult = await readBoundedBody(request, this.maxRequestBodyBytes); + context.bodyComplete = true; + if (context.cancelled) return; + if (bodyResult.kind === "too-large") { + fixedResponse(response, 413, "request body too large"); + this.finalizeRecording(pending, { + statusCode: 413, + outcome: "request-rejected", + errorReason: "request-body-too-large", + }); + return; + } + if (bodyResult.kind === "client-error") { + this.finalizeRecording(pending, { + statusCode: null, + outcome: "client-aborted", + errorReason: "client-request-error", + }); + return; + } + + let forwardedBody = await this.transformRequestBody( + bodyResult.body, + pending, + endpoint, + method, + context.abortController.signal, + ); + context.transformComplete = true; + if (context.cancelled) return; + if (forwardedBody.length > this.maxRequestBodyBytes) { + fixedResponse(response, 413, "request body too large"); + this.finalizeRecording(pending, { + statusCode: 413, + outcome: "request-rejected", + errorReason: "request-body-too-large", + }); + return; + } + let forwardedMetrics = inspectTools(forwardedBody); + if (pending) pending.metrics = forwardedMetrics; + if ( + pending?.modelCallSequence != null && + this.requiredTemperature !== undefined && + forwardedMetrics.temperature !== null && + forwardedMetrics.temperature !== this.requiredTemperature + ) { + fixedResponse(response, 400, "performance test temperature mismatch"); + this.finalizeRecording(pending, { + statusCode: 400, + outcome: "request-rejected", + errorReason: "proxy-failure", + }); + return; + } + if ( + pending?.modelCallSequence != null && + this.requiredTemperature !== undefined && + forwardedMetrics.temperature === null + ) { + const payload = JSON.parse(forwardedBody.toString("utf8")) as Record; + payload.temperature = this.requiredTemperature; + forwardedBody = Buffer.from(JSON.stringify(payload), "utf8"); + if (forwardedBody.length > this.maxRequestBodyBytes) { + fixedResponse(response, 413, "request body too large"); + this.finalizeRecording(pending, { + statusCode: 413, + outcome: "request-rejected", + errorReason: "request-body-too-large", + }); + return; + } + forwardedMetrics = inspectTools(forwardedBody); + if (pending) pending.metrics = forwardedMetrics; + } + await this.forwardRequest( + request, + response, + this.buildUpstreamUrl(incoming), + forwardedBody, + pending, + context, + ); + } catch { + fixedResponse(response, 502, "recording proxy failure"); + this.finalizeRecording(pending, { + statusCode: response.headersSent ? response.statusCode : 502, + outcome: "request-rejected", + errorReason: "proxy-failure", + }); + } finally { + clearTimeout(timer); + } + } + + private forwardRequest( + incomingRequest: IncomingMessage, + response: ServerResponse, + target: URL, + body: Buffer, + pending: PendingRecording | null, + context: ForwardContext, + ): Promise { + return new Promise((resolve) => { + let settled = false; + let statusCode: number | null = null; + let firstByteAt: number | undefined; + + const finish = (options: FinalizeOptions): void => { + if (settled) return; + settled = true; + this.finalizeRecording(pending, options); + resolve(); + }; + + const transport = target.protocol === "https:" ? https : http; + const upstreamRequest = transport.request( + target, + { + method: incomingRequest.method, + headers: forwardedRequestHeaders( + incomingRequest.headers, + body.length, + this.upstreamAuthorization, + ), + }, + (upstreamResponse) => { + statusCode = upstreamResponse.statusCode ?? 502; + if (statusCode >= 300 && statusCode < 400) { + upstreamResponse.resume(); + fixedResponse(response, 502, "upstream redirect rejected"); + finish({ + statusCode: 502, + outcome: "request-rejected", + errorReason: "proxy-failure", + }); + return; + } + if (!response.headersSent && !response.destroyed) { + response.writeHead(statusCode, forwardedResponseHeaders(upstreamResponse.headers)); + } + + upstreamResponse.once("data", () => { + firstByteAt = this.clock(); + }); + upstreamResponse.once("end", () => { + finish({ + statusCode, + firstByteAt, + outcome: "completed", + }); + }); + upstreamResponse.once("aborted", () => { + finish({ + statusCode, + firstByteAt, + outcome: context.timedOut ? "upstream-timeout" : "upstream-response-error", + errorReason: context.timedOut ? "upstream-timeout" : "upstream-response-error", + }); + if (!response.writableEnded) response.destroy(); + }); + upstreamResponse.once("error", () => { + finish({ + statusCode, + firstByteAt, + outcome: context.timedOut ? "upstream-timeout" : "upstream-response-error", + errorReason: context.timedOut ? "upstream-timeout" : "upstream-response-error", + }); + if (!response.writableEnded) response.destroy(); + }); + upstreamResponse.pipe(response); + }, + ); + context.upstreamRequest = upstreamRequest; + + response.once("close", () => { + if (settled || response.writableEnded) return; + upstreamRequest.destroy(); + finish({ + statusCode, + firstByteAt, + outcome: "client-aborted", + errorReason: "client-aborted", + }); + }); + upstreamRequest.once("error", () => { + if (context.timedOut) { + const recordedStatus = response.headersSent ? statusCode : 504; + fixedResponse(response, 504, "upstream timeout"); + finish({ + statusCode: recordedStatus, + firstByteAt, + outcome: "upstream-timeout", + errorReason: "upstream-timeout", + }); + return; + } + const recordedStatus = response.headersSent ? statusCode : 502; + fixedResponse(response, 502, "upstream unavailable"); + finish({ + statusCode: recordedStatus, + firstByteAt, + outcome: "upstream-connection-error", + errorReason: "upstream-connection-error", + }); + }); + upstreamRequest.end(body); + }); + } +} + +export function createToolDisclosureRecordingProxy( + options: RecordingProxyOptions, +): ToolDisclosureRecordingProxy { + return new ToolDisclosureRecordingProxy(options); +} diff --git a/scripts/performance/tool-disclosure/report.ts b/scripts/performance/tool-disclosure/report.ts new file mode 100644 index 0000000000..3fb46cd399 --- /dev/null +++ b/scripts/performance/tool-disclosure/report.ts @@ -0,0 +1,258 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildConservativeClaims } from "./statistics"; +import { + type ConfidenceInterval, + type EvidenceArtifact, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + type ToolDisclosureManifest, + type ToolDisclosureSummary, +} from "./types"; +import { validatePublicManifestFields } from "./validation"; + +export { buildConservativeClaims } from "./statistics"; + +export interface MarkdownReportOptions { + artifacts?: readonly EvidenceArtifact[]; +} + +/** Render a public report using only schema fields that cannot contain task content. */ +export function renderToolDisclosureMarkdown( + manifest: ToolDisclosureManifest, + summary: ToolDisclosureSummary, + options: MarkdownReportOptions = {}, +): string { + validateInputs(manifest, summary, options.artifacts ?? []); + const claims = buildConservativeClaims(summary); + if (JSON.stringify(claims) !== JSON.stringify(summary.claims)) { + throw new Error("summary claims are stale or were not generated by the claim gate"); + } + + const lines: string[] = [ + "# Progressive Tool-Disclosure Performance Test", + "", + `Schema: \`${TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION}\` `, + `Performance Test: \`${safeInline(manifest.performance_test_id)}\` `, + `Generated: ${safeInline(summary.generated_at)} `, + `SUT: \`${safeInline(manifest.sut.git_ref)}\` at \`${safeInline(manifest.sut.git_sha)}\` `, + `Harness: \`${safeInline(manifest.harness.git_ref)}\` at \`${safeInline( + manifest.harness.git_sha, + )}\``, + "", + "## Validated claims", + "", + ]; + + if (claims.length === 0) { + lines.push( + "No cross-agent improvement claim cleared all two-campaign gates. See the measured results and gate table below.", + ); + } else { + for (const claim of claims) lines.push(`- ${claim}`); + } + + lines.push( + "", + "These claims cover a deterministic synthetic tool catalog and the recorded system, model, hardware, and task corpus only. Progressive disclosure is a context optimization, not an authorization boundary. The OpenClaw native-catalog results do not establish compaction of mcporter-managed MCP tools.", + "", + "## Test environment", + "", + "| Field | Recorded value |", + "| --- | --- |", + `| Accelerator | ${cell( + manifest.environment.accelerator_count === 0 + ? "none" + : `${manifest.environment.accelerator_count} × ${manifest.environment.accelerator_type}: ${manifest.environment.accelerator_model}`, + )} |`, + `| Accelerator architecture | ${cell(manifest.environment.accelerator_architecture)} |`, + `| Accelerator driver / runtime | ${cell( + `${manifest.environment.accelerator_driver_version} / ${manifest.environment.accelerator_runtime}`, + )} |`, + `| CPU / RAM | ${cell( + `${manifest.environment.cpu_count} × ${manifest.environment.cpu_model}; ${formatNumber( + manifest.environment.ram_gib, + )} GiB`, + )} |`, + `| OS / architecture | ${cell( + `${manifest.environment.operating_system} / ${manifest.environment.architecture}`, + )} |`, + `| Model | ${cell(`${manifest.inference.model_id} @ ${manifest.inference.model_revision}`)} |`, + `| vLLM image | ${cell( + `${manifest.inference.container_image} @ ${manifest.inference.container_digest}`, + )} |`, + `| vLLM version | ${cell(manifest.inference.vllm_version)} |`, + `| Tool / reasoning parser | ${cell( + `${manifest.inference.tool_call_parser} / ${manifest.inference.reasoning_parser}`, + )} |`, + `| Public vLLM flags | ${cell(manifest.inference.public_vllm_flags.join(" ") || "(none)")} |`, + `| OpenShell | ${cell(manifest.environment.openshell_version)} |`, + `| Agent versions | ${cell( + manifest.protocol.agents + .map((agent) => `${agent}=${manifest.environment.agent_versions[agent]}`) + .join(", "), + )} |`, + `| Power / performance state | ${cell(manifest.environment.power_state)} |`, + "", + "## Exact initial schema visibility", + "", + "| Campaign | Agent | Catalog tools | Direct tokens | Progressive tokens | Reduction | Direct tools | Progressive tools |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ); + + for (const visibility of summary.static_visibility) { + lines.push( + `| ${cell(visibility.campaign_id)} | ${cell(visibility.agent)} | ${visibility.catalog_size} | ` + + `${visibility.direct.tokenizer_tokens} | ${visibility.progressive.tokenizer_tokens} | ` + + `${formatPercent(visibility.reduction.tokenizer_tokens_percent)} | ` + + `${visibility.direct.tool_count} | ${visibility.progressive.tool_count} |`, + ); + } + if (summary.static_visibility.length === 0) + lines.push("| (no measurements) | — | — | — | — | — | — | — |"); + + lines.push( + "", + "Token counts above are exact captures. If a cell has repeated captures, the summary rejects it when byte, token, or tool counts differ.", + "", + "## Paired task results", + "", + "All differences are progressive minus direct. Negative token and latency values favor progressive disclosure; positive success values favor progressive disclosure.", + "", + "| Campaign | Phase | Agent | Catalog | Paired tasks | Success Δ pp (95% CI) | Initial schema token Δ (95% CI) | Total prompt token Δ (95% CI) | First response byte Δ ms (95% CI) | E2E latency Δ ms (95% CI) |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ); + for (const result of summary.comparison_cells) { + const success = result.differences.success_percentage_points; + lines.push( + `| ${cell(result.campaign_id)} | ${cell(result.phase)} | ${cell(result.agent)} | ` + + `${result.catalog_size} | ${success.paired_tasks} | ${formatInterval(success)} | ` + + `${formatInterval(result.differences.initial_tool_schema_tokens)} | ` + + `${formatOptionalInterval(result.differences.total_prompt_tokens)} | ` + + `${formatOptionalInterval(result.differences.time_to_first_response_byte_ms)} | ` + + `${formatOptionalInterval(result.differences.end_to_end_time_ms)} |`, + ); + } + if (summary.comparison_cells.length === 0) { + lines.push("| (no measurements) | — | — | — | — | — | — | — | — | — |"); + } + + const gates = summary.claim_gates; + lines.push( + "", + "## Public claim gates", + "", + `Success noninferiority requires the paired 95% CI lower bound to be at least ${formatNumber( + gates.noninferiority_margin_percentage_points, + )} percentage points. Improvement claims require a paired 95% CI wholly below zero in each of exactly two campaigns for every named agent.`, + "", + "| Campaign | Agent | Success noninferior | Initial schema tokens improved | E2E latency improved | Reasons when blocked |", + "| --- | --- | --- | --- | --- | --- |", + ); + for (const gate of gates.campaign_agent_gates) { + lines.push( + `| ${cell(gate.campaign_id)} | ${cell(gate.agent)} | ${passFail( + gate.success_noninferior, + )} | ${passFail(gate.initial_schema_tokens_improved)} | ${passFail( + gate.end_to_end_latency_improved, + )} | ${cell(gate.reasons.join("; ") || "—")} |`, + ); + } + lines.push( + "", + `- Cross-agent success noninferiority: ${passFail(gates.cross_agent_success_noninferior)}`, + `- Cross-agent initial serialized tool-schema token improvement: ${passFail( + gates.cross_agent_initial_schema_tokens_improved, + )}`, + `- Cross-agent end-to-end latency improvement: ${passFail( + gates.cross_agent_end_to_end_latency_improved, + )}`, + "", + "## Method", + "", + `Task-level progressive/direct pairs were resampled with a deterministic paired bootstrap (${summary.bootstrap_samples.toLocaleString( + "en-US", + )} samples, seed ${summary.bootstrap_seed}). Repetitions were first averaged within each task, so tasks—not individual attempts—are the statistical unit. Setup failures are excluded; timeouts, incorrect calls, model errors, and context overflows are scored failures.`, + "", + "First-response-byte timing is reported only for streaming inference requests. It is transport TTFB, not a claim about the first decoded content token.", + "", + "The static catalog is synthetic. Exact tool names, arguments, nonces, prompts, response bodies, endpoint URLs, credentials, hostnames, usernames, and local paths are not included in this report or its numeric summary.", + ); + + const artifacts = options.artifacts ?? []; + if (artifacts.length > 0) { + lines.push( + "", + "## Evidence artifacts", + "", + "| Artifact | Kind | Bytes | SHA-256 |", + "| --- | --- | ---: | --- |", + ); + for (const artifact of artifacts) { + lines.push( + `| ${cell(artifact.file_name)} | ${cell(artifact.kind)} | ${artifact.byte_length} | \`${artifact.sha256}\` |`, + ); + } + } + + return `${lines.join("\n")}\n`; +} + +function validateInputs( + manifest: ToolDisclosureManifest, + summary: ToolDisclosureSummary, + artifacts: readonly EvidenceArtifact[], +): void { + validatePublicManifestFields(manifest); + if ( + manifest.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION || + summary.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION + ) { + throw new Error("manifest and summary must use the tool-disclosure performance test schema"); + } + if (manifest.performance_test_id !== summary.performance_test_id) { + throw new Error("manifest and summary performance test IDs do not match"); + } + for (const artifact of artifacts) { + if (artifact.file_name.includes("/") || artifact.file_name.includes("\\")) { + throw new Error(`artifact ${artifact.artifact_id} must use a leaf file name`); + } + if (!/^[a-f0-9]{64}$/.test(artifact.sha256)) { + throw new Error(`artifact ${artifact.artifact_id} has an invalid SHA-256`); + } + } +} + +function formatInterval(interval: ConfidenceInterval): string { + return `${formatNumber(interval.estimate)} [${formatNumber(interval.lower_95)}, ${formatNumber( + interval.upper_95, + )}]`; +} + +function formatOptionalInterval(interval: ConfidenceInterval | undefined): string { + return interval ? formatInterval(interval) : "—"; +} + +function formatPercent(value: number): string { + return `${formatNumber(value)}%`; +} + +function formatNumber(value: number): string { + if (Number.isInteger(value)) return String(value); + return value.toFixed(3).replace(/0+$/, "").replace(/\.$/, ""); +} + +function passFail(value: boolean): string { + return value ? "PASS" : "BLOCKED"; +} + +function safeInline(value: string): string { + return value + .replace(/[\r\n`]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function cell(value: string): string { + return safeInline(value).replaceAll("|", "\\|") || "—"; +} diff --git a/scripts/performance/tool-disclosure/run.ts b/scripts/performance/tool-disclosure/run.ts new file mode 100644 index 0000000000..8ec35d7fa8 --- /dev/null +++ b/scripts/performance/tool-disclosure/run.ts @@ -0,0 +1,759 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + artifactPath, + ensureCampaignDirectory, + readJsonLines, + scanArtifactsForForbiddenValues, + sha256File, + writeChecksumManifest, + writeJsonArtifact, + writeTextArtifact, +} from "./artifacts"; +import { + assertValidSyntheticCatalog, + DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, + generateCatalogPrefix, + generateSyntheticCatalog, + type SyntheticCatalog, +} from "./catalog"; +import { + type CompositionalRoutingAcceptanceConfig, + runCompositionalRoutingAcceptance, +} from "./compositional-tool-routing-run"; +import { + type AttemptJournalEntry, + type CampaignAttestation, + executeCampaign, + type LiveCampaignConfiguration, + type SanitizedRunEvidence, +} from "./execute"; +import { assertImmutableSandboxBase, writeOpenClawFixture } from "./openclaw-fixture"; +import { renderToolDisclosureMarkdown } from "./report"; +import { + buildToolDisclosureSchedule, + STATIC_CATALOG_SIZES, + TOOL_DISCLOSURE_AGENTS, + TOOL_DISCLOSURE_MODES, +} from "./schedule"; +import { buildToolDisclosureSummary } from "./statistics"; +import { + assertValidSyntheticTaskSet, + generatePrimaryTaskSet, + generateStressTaskSet, + type SyntheticTaskSet, +} from "./tasks"; +import { + DEFAULT_BOOTSTRAP_SAMPLES, + DEFAULT_BOOTSTRAP_SEED, + DEFAULT_NONINFERIORITY_MARGIN_PP, + type EvidenceArtifact, + type EvidenceBundle, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + type ToolDisclosureManifest, + type ToolDisclosureRun, +} from "./types"; +import { validateCompleteEvidence, validateFrozenManifest } from "./validation"; + +interface ParsedArguments { + command: "prepare" | "execute" | "summarize" | "route-acceptance" | "help"; + outputDir?: string; + sutRef: string; + catalogSeed: string; + executionSeed: number; + sandboxBase?: string; + resume: boolean; + allowDirty: boolean; + forbiddenValues: string[]; + configPath?: string; +} + +interface FixtureManifestEntry { + catalog_size: number; + files: Array<{ path: string; byte_length: number; sha256: string }>; +} + +const PREPARED_FILES = [ + "catalog.json", + "primary-tasks.json", + "stress-tasks.json", + "schedule.json", + "manifest.template.json", + "execute-config.template.json", + "fixtures.json", +] as const; + +function usage(): string { + return `Progressive tool-disclosure performance test\n\n\ +Usage:\n\ + npm run performance:tool-disclosure -- prepare --output-dir [options]\n\ + npm run performance:tool-disclosure -- execute --output-dir --config \n\ + npm run performance:tool-disclosure -- route-acceptance --output-dir --config \n\ + npm run performance:tool-disclosure -- summarize --output-dir [options]\n\n\ +Prepare options:\n\ + --sut-ref SUT revision to resolve (default: HEAD)\n\ + --catalog-seed Deterministic catalog seed\n\ + --seed Schedule seed\n\ + --sandbox-base Required immutable base image with @sha256 digest\n\ + --resume Reuse an existing output directory\n\n\ + --allow-dirty Development only; resulting evidence cannot be summarized\n\n\ +Summarize inputs:\n\ + Manifest, attestations, attempt journal, raw events, and runs in the output directory\n\ + --forbid-value Fail if a public artifact contains this value (repeatable)\n`; +} + +function requiredValue(args: readonly string[], index: number, flag: string): string { + const value = args[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`${flag} requires a value`); + return value; +} + +function parseArguments(args: readonly string[]): ParsedArguments { + const first = args[0]; + const command = + first === "prepare" || + first === "execute" || + first === "summarize" || + first === "route-acceptance" + ? first + : "help"; + if (first && command === "help" && first !== "help" && first !== "--help" && first !== "-h") { + throw new Error(`unknown command: ${first}`); + } + const parsed: ParsedArguments = { + command, + sutRef: "HEAD", + catalogSeed: DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, + executionSeed: 0x4e_56_44_41, + resume: false, + allowDirty: false, + forbiddenValues: [], + }; + for (let index = 1; index < args.length; index += 1) { + const flag = args[index]; + if (flag === "--resume") { + parsed.resume = true; + continue; + } + if (flag === "--allow-dirty") { + parsed.allowDirty = true; + continue; + } + if (flag === "--help" || flag === "-h") { + parsed.command = "help"; + continue; + } + const value = requiredValue(args, index, flag); + index += 1; + if (flag === "--output-dir") parsed.outputDir = value; + else if (flag === "--sut-ref") parsed.sutRef = value; + else if (flag === "--catalog-seed") parsed.catalogSeed = value; + else if (flag === "--sandbox-base") parsed.sandboxBase = value; + else if (flag === "--forbid-value") parsed.forbiddenValues.push(value); + else if (flag === "--config") parsed.configPath = value; + else if (flag === "--seed") { + parsed.executionSeed = Number(value); + if (!Number.isSafeInteger(parsed.executionSeed)) throw new Error("--seed must be an integer"); + } else throw new Error(`unknown option: ${flag}`); + } + if (parsed.command !== "help" && !parsed.outputDir) throw new Error("--output-dir is required"); + if (["execute", "route-acceptance"].includes(parsed.command) && !parsed.configPath) { + throw new Error(`${parsed.command} requires --config`); + } + return parsed; +} + +function gitOutput(args: readonly string[]): string { + return execFileSync("git", args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); +} + +function revision(ref: string, worktreeClean: boolean): ToolDisclosureManifest["sut"] { + const sha = gitOutput(["rev-parse", "--verify", `${ref}^{commit}`]); + if (!/^[a-f0-9]{40}$/u.test(sha)) throw new Error(`could not resolve git revision ${ref}`); + return { git_sha: sha, git_ref: ref, worktree_clean: worktreeClean }; +} + +function performanceTestId(sha: string, catalogHash: string): string { + return `tool-disclosure-${sha.slice(0, 12)}-${catalogHash.slice(0, 12)}`; +} + +function manifestTemplate(options: { + sutRef: string; + catalogHash: string; + executionSeed: number; + worktreeClean: boolean; + tasks: readonly { + id: string; + kind: ToolDisclosureManifest["protocol"]["tasks"][number]["kind"]; + }[]; +}): ToolDisclosureManifest { + const sut = revision(options.sutRef, options.worktreeClean); + const harness = revision("HEAD", options.worktreeClean); + return { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: performanceTestId(sut.git_sha, options.catalogHash), + created_at: new Date().toISOString(), + sut, + harness, + campaigns: [ + { + campaign_id: "campaign-1", + ordinal: 1, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + { + campaign_id: "campaign-2", + ordinal: 2, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + ], + protocol: { + agents: TOOL_DISCLOSURE_AGENTS, + modes: TOOL_DISCLOSURE_MODES, + catalog_sizes: STATIC_CATALOG_SIZES, + primary_catalog_size: 512, + tasks: options.tasks.map((task) => ({ + task_id: task.id, + kind: task.kind, + })), + repetitions: { "small-control": 1, primary: 5, "large-stress": 1 }, + execution_seed: options.executionSeed, + bootstrap_samples: DEFAULT_BOOTSTRAP_SAMPLES, + bootstrap_seed: DEFAULT_BOOTSTRAP_SEED, + noninferiority_margin_percentage_points: DEFAULT_NONINFERIORITY_MARGIN_PP, + retry_setup_failures: 1, + }, + environment: { + operating_system: "RECORD_BEFORE_EXECUTION", + architecture: "RECORD_BEFORE_EXECUTION", + cpu_model: "RECORD_BEFORE_EXECUTION", + cpu_count: 0, + ram_gib: 0, + accelerator_type: "RECORD_BEFORE_EXECUTION", + accelerator_model: "RECORD_BEFORE_EXECUTION", + accelerator_architecture: "RECORD_BEFORE_EXECUTION", + accelerator_count: 0, + accelerator_driver_version: "RECORD_BEFORE_EXECUTION", + accelerator_runtime: "RECORD_BEFORE_EXECUTION", + power_state: "RECORD_BEFORE_EXECUTION", + openshell_version: "RECORD_BEFORE_EXECUTION", + agent_versions: { + openclaw: "RECORD_BEFORE_EXECUTION", + hermes: "RECORD_BEFORE_EXECUTION", + "langchain-deepagents-code": "RECORD_BEFORE_EXECUTION", + }, + sandbox_image_digests: Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + "RECORD_BEFORE_EXECUTION", + ]), + ), + ), + ), + }, + inference: { + api: "chat-completions", + model_id: "RECORD_BEFORE_EXECUTION", + model_revision: "RECORD_BEFORE_EXECUTION", + container_image: "RECORD_BEFORE_EXECUTION", + container_digest: "RECORD_BEFORE_EXECUTION", + vllm_version: "RECORD_BEFORE_EXECUTION", + tool_call_parser: "RECORD_BEFORE_EXECUTION", + reasoning_parser: "RECORD_BEFORE_EXECUTION", + temperature: 0, + concurrency: 1, + prefix_caching_enabled: false, + public_vllm_flags: [], + }, + }; +} + +function prepare(options: ParsedArguments): void { + const repositoryRoot = path.resolve(gitOutput(["rev-parse", "--show-toplevel"])); + const requestedOutput = path.resolve(options.outputDir as string); + if ( + requestedOutput === repositoryRoot || + requestedOutput.startsWith(`${repositoryRoot}${path.sep}`) + ) { + throw new Error("performance test output directory must be outside the Git worktree"); + } + const worktreeClean = gitOutput(["status", "--porcelain=v1", "--untracked-files=normal"]) === ""; + if (!worktreeClean && !options.allowDirty) { + throw new Error( + "performance test preparation requires a clean worktree; commit or stash changes first", + ); + } + if (!options.sandboxBase) { + throw new Error("--sandbox-base is required and must end in an immutable @sha256 digest"); + } + assertImmutableSandboxBase(options.sandboxBase); + const outputDir = ensureCampaignDirectory(requestedOutput, options.resume); + const catalog = generateSyntheticCatalog({ seed: options.catalogSeed }); + const primary = generatePrimaryTaskSet(catalog); + const stress = generateStressTaskSet(catalog); + const schedule = buildToolDisclosureSchedule({ + primaryTaskIds: primary.tasks.map((task) => task.id), + stressTaskIds: stress.tasks.map((task) => task.id), + seed: options.executionSeed, + }); + const manifest = manifestTemplate({ + sutRef: options.sutRef, + catalogHash: catalog.tools_sha256, + executionSeed: options.executionSeed, + worktreeClean, + tasks: [...primary.tasks, ...stress.tasks], + }); + writeJsonArtifact(outputDir, "catalog.json", catalog); + writeJsonArtifact(outputDir, "primary-tasks.json", primary); + writeJsonArtifact(outputDir, "stress-tasks.json", stress); + writeJsonArtifact(outputDir, "schedule.json", schedule); + writeJsonArtifact(outputDir, "manifest.template.json", manifest); + writeJsonArtifact(outputDir, "execute-config.template.json", { + campaign: 1, + upstream_vllm_url: "http://127.0.0.1:8000", + telemetry_url: "http://127.0.0.1:8000", + tokenizer_model: "REPLACE_WITH_MODEL_ID", + recorder_port: 18_080, + managed_inference_base_url: "http://127.0.0.1:18080", + vllm_container_name: "replace-vllm-container-name", + vllm_container_id: "replace-vllm-container-id", + sandbox_names: Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + `replace-${agent}-${mode}-${size}`, + ]), + ), + ), + ), + sandbox_instance_ids: Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + `replace-live-instance-id-${agent}-${mode}-${size}`, + ]), + ), + ), + ), + sandbox_container_names: Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + `replace-live-container-${agent}-${mode}-${size}`, + ]), + ), + ), + ), + timeout_ms: 600_000, + } satisfies LiveCampaignConfiguration); + + for (const size of STATIC_CATALOG_SIZES) { + const fixtureDir = path.join(outputDir, `openclaw-${size}`); + if (fs.existsSync(fixtureDir) && fs.readdirSync(fixtureDir).length > 0) { + const existing = JSON.parse( + fs.readFileSync(path.join(fixtureDir, "plugin", "catalog.json"), "utf8"), + ) as { tools_sha256?: string }; + const expected = generateCatalogPrefix(catalog, size); + if (existing.tools_sha256 !== expected.tools_sha256) { + throw new Error(`existing OpenClaw fixture ${size} does not match the prepared catalog`); + } + continue; + } + writeOpenClawFixture({ + outputDir: fixtureDir, + catalog: generateCatalogPrefix(catalog, size), + sandboxBase: options.sandboxBase, + }); + } + writeJsonArtifact( + outputDir, + "fixtures.json", + STATIC_CATALOG_SIZES.map((size) => { + const fixtureRoot = path.join(outputDir, `openclaw-${size}`); + const files = [ + ".dockerignore", + "Dockerfile", + "plugin/catalog.json", + "plugin/index.js", + "plugin/openclaw.plugin.json", + "plugin/package.json", + ]; + return { + catalog_size: size, + files: files.map((name) => { + const file = path.join(fixtureRoot, name); + return { + path: `openclaw-${size}/${name}`, + byte_length: fs.statSync(file).size, + sha256: sha256File(file), + }; + }), + }; + }), + ); + writeChecksumManifest(outputDir, PREPARED_FILES); + process.stdout.write( + `Prepared ${schedule.length} runs in ${outputDir}. Copy manifest.template.json to manifest.json and replace every RECORD_BEFORE_EXECUTION value before execution.\n`, + ); +} + +function readJson(file: string): T { + try { + return JSON.parse(fs.readFileSync(file, "utf8")) as T; + } catch (error) { + throw new Error(`could not read ${path.basename(file)}: ${String(error)}`); + } +} + +function evidenceArtifact( + outputDir: string, + fileName: string, + kind: EvidenceArtifact["kind"], +): EvidenceArtifact { + const file = artifactPath(outputDir, fileName); + return { + artifact_id: `${kind}-${fileName}`, + kind, + file_name: fileName, + media_type: fileName.endsWith(".jsonl") + ? "application/x-ndjson" + : fileName.endsWith(".md") + ? "text/markdown" + : "application/json", + byte_length: fs.statSync(file).size, + sha256: sha256File(file), + }; +} + +function assertManifestCompleted(manifest: ToolDisclosureManifest): void { + const serialized = JSON.stringify(manifest); + if (serialized.includes("RECORD_BEFORE_EXECUTION")) { + throw new Error("manifest.json still contains RECORD_BEFORE_EXECUTION placeholders"); + } +} + +function assertPreparedIdentity( + manifest: ToolDisclosureManifest, + catalog: SyntheticCatalog, + primaryTasks: SyntheticTaskSet, + stressTasks: SyntheticTaskSet, +): void { + if ( + manifest.performance_test_id !== performanceTestId(manifest.sut.git_sha, catalog.tools_sha256) + ) { + throw new Error("manifest performance test ID does not match the frozen SUT and catalog"); + } + const expectedTasks = [...primaryTasks.tasks, ...stressTasks.tasks].map((task) => ({ + task_id: task.id, + kind: task.kind, + })); + if (JSON.stringify(manifest.protocol.tasks) !== JSON.stringify(expectedTasks)) { + throw new Error("manifest tasks do not match the prepared task artifacts"); + } +} + +export function validateCampaignAttestations( + manifest: ToolDisclosureManifest, + attestations: readonly CampaignAttestation[], +): void { + if ( + attestations.length !== 2 || + JSON.stringify(attestations.map((item) => item.campaign_id).sort()) !== + JSON.stringify(["campaign-1", "campaign-2"]) + ) { + throw new Error("evidence requires exactly two campaign attestations"); + } + if (new Set(attestations.map((item) => item.vllm_process_start_time_seconds)).size !== 2) { + throw new Error("campaign attestations reused a vLLM process"); + } + if ( + new Set(attestations.map((item) => item.inference_container_id_sha256)).size !== 2 || + new Set(attestations.map((item) => item.inference_config_sha256)).size !== 1 || + attestations.some( + (item) => + !/^[a-f0-9]{64}$/u.test(item.inference_container_id_sha256) || + !/^[a-f0-9]{64}$/u.test(item.inference_config_sha256) || + item.inference_image_digest !== manifest.inference.container_digest, + ) + ) { + throw new Error("campaigns require distinct vLLM containers with identical frozen config"); + } + const allInstanceIds = attestations.flatMap((item) => + item.sandbox_cells.map((cell) => cell.instance_id_sha256), + ); + const expectedCells = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => `${agent}:${mode}:${size}`), + ), + ).sort(); + if ( + attestations.some( + (item) => + !Number.isFinite(item.vllm_process_start_time_seconds) || + item.vllm_process_start_time_seconds <= 0 || + JSON.stringify(item.sandbox_cells.map((cell) => cell.cell).sort()) !== + JSON.stringify(expectedCells) || + item.sandbox_cells.some( + (cell) => + !/^[a-f0-9]{64}$/u.test(cell.instance_id_sha256) || + !/^[a-f0-9]{64}$/u.test(cell.status_sha256), + ), + ) || + new Set(allInstanceIds).size !== 60 + ) { + throw new Error("campaign attestations reused or omitted sandbox instances"); + } + for (const attestation of attestations) { + for (const cell of attestation.sandbox_cells) { + if (manifest.environment.sandbox_image_digests?.[cell.cell] !== cell.image_digest) { + throw new Error(`campaign attestation image mismatch for ${cell.cell}`); + } + } + } +} + +function validateOpenClawFixtures( + outputDir: string, + catalog: SyntheticCatalog, + fixtureManifest: readonly FixtureManifestEntry[], +): void { + if ( + !sameNumberList( + fixtureManifest.map((entry) => entry.catalog_size), + STATIC_CATALOG_SIZES, + ) + ) { + throw new Error("fixtures.json does not contain the frozen catalog sizes"); + } + for (const entry of fixtureManifest) { + const root = path.join(outputDir, `openclaw-${entry.catalog_size}`); + const dockerfile = fs.readFileSync(path.join(root, "Dockerfile"), "utf8"); + const sandboxBase = dockerfile.match(/^ARG SANDBOX_BASE=(\S+)$/mu)?.[1]; + if (!sandboxBase) throw new Error(`OpenClaw ${entry.catalog_size} fixture has no base image`); + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fixture-verify-")); + const expectedRoot = path.join(temporaryRoot, `fixture-${entry.catalog_size}`); + try { + const generated = writeOpenClawFixture({ + outputDir: expectedRoot, + catalog: generateCatalogPrefix(catalog, entry.catalog_size), + sandboxBase, + }); + if (entry.files.length !== generated.files.length) { + throw new Error(`OpenClaw ${entry.catalog_size} fixture file count changed`); + } + for (const name of generated.files) { + const relative = `openclaw-${entry.catalog_size}/${name}`; + const recorded = entry.files.find((file) => file.path === relative); + const actual = path.join(root, name); + const expected = path.join(expectedRoot, name); + if ( + !recorded || + recorded.byte_length !== fs.statSync(actual).size || + recorded.sha256 !== sha256File(actual) || + recorded.sha256 !== sha256File(expected) + ) { + throw new Error(`OpenClaw fixture integrity check failed for ${relative}`); + } + } + } finally { + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } + } +} + +function sameNumberList(left: readonly number[], right: readonly number[]): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +function summarize(options: ParsedArguments): void { + const outputDir = path.resolve(options.outputDir as string); + const manifest = readJson(artifactPath(outputDir, "manifest.json")); + assertManifestCompleted(manifest); + validateFrozenManifest(manifest); + const catalog = readJson(artifactPath(outputDir, "catalog.json")); + const primaryTasks = readJson(artifactPath(outputDir, "primary-tasks.json")); + const stressTasks = readJson(artifactPath(outputDir, "stress-tasks.json")); + const fixtures = readJson(artifactPath(outputDir, "fixtures.json")); + const schedule = readJson>( + artifactPath(outputDir, "schedule.json"), + ); + assertValidSyntheticCatalog(catalog); + assertValidSyntheticTaskSet(primaryTasks, catalog); + assertValidSyntheticTaskSet(stressTasks, catalog); + assertPreparedIdentity(manifest, catalog, primaryTasks, stressTasks); + validateOpenClawFixtures(outputDir, catalog, fixtures); + const runs = readJsonLines(artifactPath(outputDir, "runs.jsonl")); + const rawEvidence = readJsonLines( + artifactPath(outputDir, "raw-events.jsonl"), + ); + const journal = readJsonLines( + artifactPath(outputDir, "attempt-journal.jsonl"), + ); + if ( + JSON.stringify(journal.map((entry) => entry.raw)) !== JSON.stringify(rawEvidence) || + JSON.stringify(journal.map((entry) => entry.run)) !== JSON.stringify(runs) + ) { + throw new Error("materialized evidence differs from the atomic attempt journal"); + } + const attestations = readJson( + artifactPath(outputDir, "campaign-attestations.json"), + ); + validateCampaignAttestations(manifest, attestations); + validateCompleteEvidence({ + manifest, + runs, + schedule, + primaryTasks, + stressTasks, + rawEvidence, + catalog, + }); + const summary = buildToolDisclosureSummary(manifest, runs); + writeJsonArtifact(outputDir, "summary.json", summary); + const reportInputs = [ + evidenceArtifact(outputDir, "catalog.json", "catalog"), + evidenceArtifact(outputDir, "primary-tasks.json", "tasks"), + evidenceArtifact(outputDir, "stress-tasks.json", "tasks"), + evidenceArtifact(outputDir, "schedule.json", "schedule"), + evidenceArtifact(outputDir, "fixtures.json", "fixture-manifest"), + evidenceArtifact(outputDir, "manifest.json", "manifest"), + evidenceArtifact(outputDir, "raw-events.jsonl", "raw-events"), + evidenceArtifact(outputDir, "attempt-journal.jsonl", "attempt-journal"), + evidenceArtifact(outputDir, "campaign-attestations.json", "attestation"), + evidenceArtifact(outputDir, "runs.jsonl", "runs"), + evidenceArtifact(outputDir, "summary.json", "summary"), + ]; + writeTextArtifact( + outputDir, + "report.md", + renderToolDisclosureMarkdown(manifest, summary, { + artifacts: reportInputs, + }), + ); + const artifacts = [...reportInputs, evidenceArtifact(outputDir, "report.md", "report")]; + const bundle: EvidenceBundle = { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: manifest.performance_test_id, + generated_at: new Date().toISOString(), + artifacts, + }; + writeJsonArtifact(outputDir, "evidence.json", bundle); + const publicNames = [ + "catalog.json", + "primary-tasks.json", + "stress-tasks.json", + "schedule.json", + "fixtures.json", + "manifest.json", + "raw-events.jsonl", + "attempt-journal.jsonl", + "campaign-attestations.json", + "runs.jsonl", + "summary.json", + "report.md", + "evidence.json", + ]; + scanArtifactsForForbiddenValues(outputDir, publicNames, options.forbiddenValues); + writeChecksumManifest(outputDir, publicNames); + process.stdout.write( + `Summarized ${runs.length} runs. ${summary.claims.length} claim(s) cleared all gates.\n`, + ); +} + +async function execute(options: ParsedArguments): Promise { + const outputDir = path.resolve(options.outputDir as string); + const manifest = readJson(artifactPath(outputDir, "manifest.json")); + assertManifestCompleted(manifest); + validateFrozenManifest(manifest); + if ( + gitOutput(["rev-parse", "HEAD"]) !== manifest.harness.git_sha || + gitOutput(["status", "--porcelain=v1", "--untracked-files=normal"]) !== "" + ) { + throw new Error("execute requires the clean harness HEAD recorded in manifest.json"); + } + const catalog = readJson(artifactPath(outputDir, "catalog.json")); + const primaryTasks = readJson(artifactPath(outputDir, "primary-tasks.json")); + const stressTasks = readJson(artifactPath(outputDir, "stress-tasks.json")); + const schedule = readJson>( + artifactPath(outputDir, "schedule.json"), + ); + assertValidSyntheticCatalog(catalog); + assertValidSyntheticTaskSet(primaryTasks, catalog); + assertValidSyntheticTaskSet(stressTasks, catalog); + assertPreparedIdentity(manifest, catalog, primaryTasks, stressTasks); + const expectedSchedule = buildToolDisclosureSchedule({ + primaryTaskIds: primaryTasks.tasks.map((task) => task.id), + stressTaskIds: stressTasks.tasks.map((task) => task.id), + seed: manifest.protocol.execution_seed, + }); + if (JSON.stringify(schedule) !== JSON.stringify(expectedSchedule)) { + throw new Error("execute refused a modified schedule.json"); + } + await executeCampaign({ + outputDir, + config: readJson(path.resolve(options.configPath as string)), + manifest, + catalog, + primaryTasks, + stressTasks, + schedule, + }); +} + +async function routeAcceptance(options: ParsedArguments): Promise { + const outputDir = ensureCampaignDirectory( + path.resolve(options.outputDir as string), + options.resume, + ); + const config = readJson( + path.resolve(options.configPath as string), + ); + const output = await runCompositionalRoutingAcceptance(config); + const artifactName = "compositional-routing-acceptance.json"; + writeJsonArtifact(outputDir, artifactName, output); + scanArtifactsForForbiddenValues(outputDir, [artifactName], options.forbiddenValues); + writeChecksumManifest(outputDir, [artifactName]); + process.stdout.write( + `Compositional routing acceptance ${output.acceptance_passed ? "passed" : "failed"}.\n`, + ); + if (!output.acceptance_passed) { + throw new Error("compositional routing acceptance gates did not pass"); + } +} + +export async function main(args = process.argv.slice(2)): Promise { + const options = parseArguments(args); + if (options.command === "help") { + process.stdout.write(usage()); + return; + } + if (options.command === "prepare") prepare(options); + else if (options.command === "execute") await execute(options); + else if (options.command === "route-acceptance") await routeAcceptance(options); + else summarize(options); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + void main().catch((error: unknown) => { + process.stderr.write( + `tool-disclosure performance test: ${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exitCode = 1; + }); +} diff --git a/scripts/performance/tool-disclosure/schedule.ts b/scripts/performance/tool-disclosure/schedule.ts new file mode 100644 index 0000000000..a588f3fbc7 --- /dev/null +++ b/scripts/performance/tool-disclosure/schedule.ts @@ -0,0 +1,225 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { PRIMARY_TASK_COUNT, STRESS_TASK_COUNT } from "./tasks"; + +export const TOOL_DISCLOSURE_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +export const TOOL_DISCLOSURE_MODES = ["progressive", "direct"] as const; +export const STATIC_CATALOG_SIZES = [16, 64, 256, 512, 2_209] as const; +const SMALL_CONTROL_CATALOG_SIZE = STATIC_CATALOG_SIZES[1]; +const PRIMARY_CATALOG_SIZE = STATIC_CATALOG_SIZES[3]; +const LARGE_STRESS_CATALOG_SIZE = STATIC_CATALOG_SIZES[4]; + +export type ToolDisclosureAgent = (typeof TOOL_DISCLOSURE_AGENTS)[number]; +export type ToolDisclosureMode = (typeof TOOL_DISCLOSURE_MODES)[number]; +export type ToolDisclosurePhase = + | "static-visibility" + | "small-control" + | "primary" + | "large-stress"; + +export interface ScheduledToolDisclosureRun { + run_id: string; + campaign: 1 | 2; + phase: ToolDisclosurePhase; + agent: ToolDisclosureAgent; + mode: ToolDisclosureMode; + catalog_size: number; + repetition: number; + task_id?: string; + pair_id: string; + scored: boolean; +} + +export interface ToolDisclosureScheduleOptions { + primaryTaskIds: readonly string[]; + stressTaskIds: readonly string[]; + seed: number; + campaigns?: readonly (1 | 2)[]; +} + +interface PairBlock { + sortKey: number; + runs: [ScheduledToolDisclosureRun, ScheduledToolDisclosureRun]; +} + +function xorshift32(value: number): number { + let state = value | 0; + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return state >>> 0; +} + +function stableHash(text: string, seed: number): number { + let state = seed >>> 0; + for (const char of text) { + state = xorshift32(state ^ (char.codePointAt(0) ?? 0)); + } + return state; +} + +function runId( + campaign: number, + phase: ToolDisclosurePhase, + agent: ToolDisclosureAgent, + mode: ToolDisclosureMode, + catalogSize: number, + taskId: string | undefined, + repetition: number, +): string { + return [ + `c${campaign}`, + phase, + agent, + mode, + `n${catalogSize}`, + taskId ?? "capture", + `r${repetition}`, + ].join("--"); +} + +function makePair( + campaign: 1 | 2, + phase: ToolDisclosurePhase, + agent: ToolDisclosureAgent, + catalogSize: number, + taskId: string | undefined, + repetition: number, + seed: number, +): PairBlock { + const pairId = [ + `c${campaign}`, + phase, + agent, + `n${catalogSize}`, + taskId ?? "capture", + `r${repetition}`, + ].join("--"); + const modes: [ToolDisclosureMode, ToolDisclosureMode] = + stableHash(pairId, seed) % 2 === 0 ? ["progressive", "direct"] : ["direct", "progressive"]; + const scored = phase !== "static-visibility"; + return { + sortKey: stableHash(`sort:${pairId}`, seed), + runs: modes.map((mode) => ({ + run_id: runId(campaign, phase, agent, mode, catalogSize, taskId, repetition), + campaign, + phase, + agent, + mode, + catalog_size: catalogSize, + repetition, + ...(taskId ? { task_id: taskId } : {}), + pair_id: pairId, + scored, + })) as [ScheduledToolDisclosureRun, ScheduledToolDisclosureRun], + }; +} + +function assertUniqueNonEmpty(values: readonly string[], label: string): void { + if (values.length === 0) throw new Error(`${label} must not be empty`); + const normalized = values.map((value) => value.trim()); + if (normalized.some((value) => !value)) throw new Error(`${label} contains an empty id`); + if (new Set(normalized).size !== normalized.length) + throw new Error(`${label} contains duplicate ids`); +} + +export function buildToolDisclosureSchedule( + options: ToolDisclosureScheduleOptions, +): ScheduledToolDisclosureRun[] { + assertUniqueNonEmpty(options.primaryTaskIds, "primaryTaskIds"); + assertUniqueNonEmpty(options.stressTaskIds, "stressTaskIds"); + if (options.primaryTaskIds.length !== PRIMARY_TASK_COUNT) { + throw new Error( + `primaryTaskIds must contain exactly ${PRIMARY_TASK_COUNT} tasks, got ${options.primaryTaskIds.length}`, + ); + } + if (options.stressTaskIds.length !== STRESS_TASK_COUNT) { + throw new Error( + `stressTaskIds must contain exactly ${STRESS_TASK_COUNT} tasks, got ${options.stressTaskIds.length}`, + ); + } + if (!Number.isSafeInteger(options.seed)) throw new Error("seed must be a safe integer"); + + const campaigns = options.campaigns ?? [1, 2]; + if (campaigns.length === 0 || campaigns.some((value) => value !== 1 && value !== 2)) { + throw new Error("campaigns must contain campaign 1, campaign 2, or both"); + } + + const scheduled: ScheduledToolDisclosureRun[] = []; + for (const campaign of campaigns) { + const staticBlocks: PairBlock[] = []; + const liveBlocks: PairBlock[] = []; + for (const agent of TOOL_DISCLOSURE_AGENTS) { + for (const catalogSize of STATIC_CATALOG_SIZES) { + staticBlocks.push( + makePair(campaign, "static-visibility", agent, catalogSize, undefined, 1, options.seed), + ); + } + for (const taskId of options.primaryTaskIds) { + liveBlocks.push( + makePair( + campaign, + "small-control", + agent, + SMALL_CONTROL_CATALOG_SIZE, + taskId, + 1, + options.seed, + ), + ); + for (let repetition = 1; repetition <= 5; repetition += 1) { + liveBlocks.push( + makePair( + campaign, + "primary", + agent, + PRIMARY_CATALOG_SIZE, + taskId, + repetition, + options.seed, + ), + ); + } + } + for (const taskId of options.stressTaskIds) { + liveBlocks.push( + makePair( + campaign, + "large-stress", + agent, + LARGE_STRESS_CATALOG_SIZE, + taskId, + 1, + options.seed, + ), + ); + } + } + staticBlocks.sort( + (a, b) => a.sortKey - b.sortKey || a.runs[0].pair_id.localeCompare(b.runs[0].pair_id), + ); + liveBlocks.sort( + (a, b) => a.sortKey - b.sortKey || a.runs[0].pair_id.localeCompare(b.runs[0].pair_id), + ); + scheduled.push(...staticBlocks.flatMap((block) => block.runs)); + scheduled.push(...liveBlocks.flatMap((block) => block.runs)); + } + + const ids = scheduled.map((run) => run.run_id); + if (new Set(ids).size !== ids.length) + throw new Error("generated schedule contains duplicate run ids"); + return scheduled; +} + +export function countScheduledRunsByCampaign( + schedule: readonly ScheduledToolDisclosureRun[], +): Record { + const counts: Record = {}; + for (const run of schedule) { + const key = `campaign-${run.campaign}:${run.phase}`; + counts[key] = (counts[key] ?? 0) + 1; + } + return counts; +} diff --git a/scripts/performance/tool-disclosure/smoke-mcp-transport.ts b/scripts/performance/tool-disclosure/smoke-mcp-transport.ts new file mode 100644 index 0000000000..078070ba08 --- /dev/null +++ b/scripts/performance/tool-disclosure/smoke-mcp-transport.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const PERFORMANCE_SMOKE_MCP_URL_ENV = "NEMOCLAW_TOOL_DISCLOSURE_PERFORMANCE_MCP_URL"; +export const PERFORMANCE_SMOKE_MCP_PORT_ENV = "NEMOCLAW_TOOL_DISCLOSURE_PERFORMANCE_MCP_PORT"; + +export type PerformanceSmokeMcpTransport = + | { kind: "local-quick-tunnel" } + | { kind: "external-ssh-forward"; mcpUrl: string; listenPort: number }; + +function externalMcpUrl(value: string): string { + const url = new URL(value); + if ( + url.protocol !== "https:" || + url.username !== "" || + url.password !== "" || + url.port !== "" || + url.search !== "" || + url.hash !== "" || + url.pathname !== "/mcp" || + !/^[a-z0-9-]+\.trycloudflare\.com$/u.test(url.hostname) + ) { + throw new Error("external performance smoke MCP URL must be an exact trycloudflare /mcp URL"); + } + return url.toString(); +} + +export function resolvePerformanceSmokeMcpTransport( + env: NodeJS.ProcessEnv = process.env, +): PerformanceSmokeMcpTransport { + const rawUrl = env[PERFORMANCE_SMOKE_MCP_URL_ENV]?.trim() ?? ""; + const rawPort = env[PERFORMANCE_SMOKE_MCP_PORT_ENV]?.trim() ?? ""; + if (!rawUrl && !rawPort) return { kind: "local-quick-tunnel" }; + if (!rawUrl || !rawPort) { + throw new Error("external performance smoke MCP URL and port must be configured together"); + } + if (!/^[0-9]{1,5}$/u.test(rawPort)) { + throw new Error("external performance smoke MCP port must be an integer"); + } + const listenPort = Number(rawPort); + if (listenPort < 1_024 || listenPort > 65_535) { + throw new Error("external performance smoke MCP port must be between 1024 and 65535"); + } + return { + kind: "external-ssh-forward", + mcpUrl: externalMcpUrl(rawUrl), + listenPort, + }; +} + +export async function waitForPerformanceSmokeMcpEndpoint( + mcpUrl: string, + options: { fetchImpl?: typeof fetch; timeoutMs?: number } = {}, +): Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const deadline = Date.now() + (options.timeoutMs ?? 30_000); + while (Date.now() < deadline) { + try { + const response = await fetchImpl(mcpUrl, { + method: "HEAD", + redirect: "error", + signal: AbortSignal.timeout(5_000), + }); + await response.body?.cancel(); + if (response.status === 405) return; + } catch { + // The relay and server handoff are allowed a short convergence window. + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error("performance smoke MCP endpoint did not become ready before the timeout"); +} diff --git a/scripts/performance/tool-disclosure/statistics.ts b/scripts/performance/tool-disclosure/statistics.ts new file mode 100644 index 0000000000..fd0d637abf --- /dev/null +++ b/scripts/performance/tool-disclosure/statistics.ts @@ -0,0 +1,739 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + type PerformanceTestPhase, + type CampaignAgentGate, + type ClaimGateSummary, + type ComparisonCellSummary, + type ConfidenceInterval, + DEFAULT_BOOTSTRAP_SAMPLES, + DEFAULT_BOOTSTRAP_SEED, + DEFAULT_NONINFERIORITY_MARGIN_PP, + type ModeAggregate, + type SchemaReductionSummary, + type SchemaVisibilitySnapshot, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + type ToolDisclosureAgent, + type ToolDisclosureManifest, + type ToolDisclosureMode, + type ToolDisclosureRun, + type ToolDisclosureSummary, +} from "./types"; + +export interface PairedObservation { + progressive: number; + direct: number; +} + +export interface PairedBootstrapOptions { + samples?: number; + seed?: number; +} + +export interface ComparisonOptions extends PairedBootstrapOptions {} + +export interface ClaimGateOptions { + agents: readonly ToolDisclosureAgent[]; + campaignIds: readonly string[]; + primaryCatalogSize: number; + noninferiorityMarginPercentagePoints?: number; +} + +interface TaskAggregate { + success: number; + initialSchemaTokens: number; + totalPromptTokens?: number; + timeToFirstResponseByteMs?: number; + endToEndTimeMs?: number; +} + +interface ComparisonGroup { + campaignId: string; + phase: Exclude; + agent: ToolDisclosureAgent; + catalogSize: number; + runs: ToolDisclosureRun[]; +} + +interface StaticGroup { + campaignId: string; + agent: ToolDisclosureAgent; + catalogSize: number; + runs: ToolDisclosureRun[]; +} + +const PHASE_ORDER: Readonly, number>> = { + "small-control": 0, + primary: 1, + "large-stress": 2, +}; + +/** + * Percentile paired bootstrap over task-level observations. Each bootstrap + * draw resamples whole progressive/direct task pairs, never individual runs. + */ +export function pairedBootstrapDifference( + observations: readonly PairedObservation[], + options: PairedBootstrapOptions = {}, +): ConfidenceInterval { + const samples = options.samples ?? DEFAULT_BOOTSTRAP_SAMPLES; + const seed = normalizeSeed(options.seed ?? DEFAULT_BOOTSTRAP_SEED); + if (!Number.isInteger(samples) || samples <= 0) { + throw new Error(`bootstrap samples must be a positive integer, received ${samples}`); + } + if (observations.length === 0) { + throw new Error("paired bootstrap requires at least one task pair"); + } + for (const observation of observations) { + assertFinite(observation.progressive, "progressive observation"); + assertFinite(observation.direct, "direct observation"); + } + + const differences = observations.map(({ progressive, direct }) => progressive - direct); + const estimate = mean(differences); + const random = mulberry32(seed); + const bootstrapEstimates = new Array(samples); + for (let sampleIndex = 0; sampleIndex < samples; sampleIndex += 1) { + let total = 0; + for (let draw = 0; draw < differences.length; draw += 1) { + total += differences[Math.floor(random() * differences.length)]; + } + bootstrapEstimates[sampleIndex] = total / differences.length; + } + bootstrapEstimates.sort((left, right) => left - right); + + return { + estimate: round6(estimate), + lower_95: round6(quantile(bootstrapEstimates, 0.025)), + upper_95: round6(quantile(bootstrapEstimates, 0.975)), + paired_tasks: observations.length, + bootstrap_samples: samples, + bootstrap_seed: seed, + }; +} + +export function buildComparisonCells( + runs: readonly ToolDisclosureRun[], + options: ComparisonOptions = {}, +): ComparisonCellSummary[] { + const samples = options.samples ?? DEFAULT_BOOTSTRAP_SAMPLES; + const seed = normalizeSeed(options.seed ?? DEFAULT_BOOTSTRAP_SEED); + const groups = new Map(); + + for (const run of runs) { + validateRun(run); + if (run.phase === "static-visibility") continue; + const key = [run.campaign_id, run.phase, run.agent, run.catalog_size].join("\u0000"); + const group = groups.get(key) ?? { + campaignId: run.campaign_id, + phase: run.phase, + agent: run.agent, + catalogSize: run.catalog_size, + runs: [], + }; + group.runs.push(run); + groups.set(key, group); + } + + return [...groups.values()] + .sort(compareComparisonGroups) + .map((group) => buildComparisonCell(group, samples, seed)); +} + +/** + * Produces exact visibility reductions. Repeated captures are accepted only + * when every count is identical, preventing a nondeterministic schema snapshot + * from being rounded into a public claim. + */ +export function summarizeStaticVisibility( + runs: readonly ToolDisclosureRun[], +): SchemaReductionSummary[] { + const groups = new Map(); + for (const run of runs) { + validateRun(run); + if (run.phase !== "static-visibility") continue; + const key = [run.campaign_id, run.agent, run.catalog_size].join("\u0000"); + const group = groups.get(key) ?? { + campaignId: run.campaign_id, + agent: run.agent, + catalogSize: run.catalog_size, + runs: [], + }; + group.runs.push(run); + groups.set(key, group); + } + + return [...groups.values()].sort(compareStaticGroups).map((group) => { + const direct = exactVisibilitySnapshot(group, "direct"); + const progressive = exactVisibilitySnapshot(group, "progressive"); + if (direct.tokenizer_tokens <= 0) { + throw new Error( + `direct static visibility must contain tokenizer tokens for ${staticGroupLabel(group)}`, + ); + } + return { + campaign_id: group.campaignId, + agent: group.agent, + catalog_size: group.catalogSize, + direct, + progressive, + reduction: { + tool_count: direct.tool_count - progressive.tool_count, + serialized_bytes: direct.serialized_bytes - progressive.serialized_bytes, + tokenizer_tokens: direct.tokenizer_tokens - progressive.tokenizer_tokens, + tokenizer_tokens_percent: round6( + ((direct.tokenizer_tokens - progressive.tokenizer_tokens) / direct.tokenizer_tokens) * + 100, + ), + }, + }; + }); +} + +export function evaluateClaimGates( + comparisonCells: readonly ComparisonCellSummary[], + staticVisibility: readonly SchemaReductionSummary[], + options: ClaimGateOptions, +): ClaimGateSummary { + const agents = unique(options.agents); + const campaignIds = unique(options.campaignIds); + const margin = options.noninferiorityMarginPercentagePoints ?? DEFAULT_NONINFERIORITY_MARGIN_PP; + assertFinite(margin, "noninferiority margin"); + const campaignCountValid = campaignIds.length === 2; + const gates: CampaignAgentGate[] = []; + + for (const campaignId of campaignIds) { + for (const agent of agents) { + const matches = comparisonCells.filter( + (cell) => + cell.campaign_id === campaignId && + cell.phase === "primary" && + cell.agent === agent && + cell.catalog_size === options.primaryCatalogSize, + ); + const staticMatches = staticVisibility.filter( + (cell) => + cell.campaign_id === campaignId && + cell.agent === agent && + cell.catalog_size === options.primaryCatalogSize, + ); + const reasons: string[] = []; + if (!campaignCountValid) reasons.push("claim gate requires exactly two campaigns"); + if (matches.length !== 1) { + reasons.push(`expected one primary comparison cell, found ${matches.length}`); + } + if (staticMatches.length !== 1) { + reasons.push( + `expected one deterministic static visibility cell, found ${staticMatches.length}`, + ); + } + + const cell = matches[0]; + const staticCell = staticMatches[0]; + const successLower = cell?.differences.success_percentage_points.lower_95; + const schemaUpper = cell?.differences.initial_tool_schema_tokens.upper_95; + const latencyUpper = cell?.differences.end_to_end_time_ms?.upper_95; + const successNoninferior = + campaignCountValid && + matches.length === 1 && + successLower !== undefined && + cell.differences.success_percentage_points.paired_tasks === 24 && + successLower >= margin; + const schemaImproved = + campaignCountValid && + successNoninferior && + matches.length === 1 && + staticMatches.length === 1 && + schemaUpper !== undefined && + cell.differences.initial_tool_schema_tokens.paired_tasks === 24 && + schemaUpper < 0 && + staticCell.reduction.tokenizer_tokens > 0; + const latencyImproved = + campaignCountValid && + successNoninferior && + matches.length === 1 && + latencyUpper !== undefined && + cell.differences.end_to_end_time_ms?.paired_tasks === 24 && + latencyUpper < 0; + + if (cell && !successNoninferior) { + if (cell.differences.success_percentage_points.paired_tasks !== 24) { + reasons.push("success gate requires exactly 24 paired primary tasks"); + } else { + reasons.push( + `success lower CI ${formatNumber(successLower)} pp is below ${formatNumber(margin)} pp`, + ); + } + } + if (cell && staticCell && !schemaImproved) { + reasons.push( + cell.differences.initial_tool_schema_tokens.paired_tasks !== 24 + ? "schema-token gate requires exactly 24 paired primary tasks" + : "initial schema-token improvement is not significant and repeatable", + ); + } + if (cell && !latencyImproved) { + reasons.push( + cell.differences.end_to_end_time_ms?.paired_tasks !== 24 + ? "latency gate requires exactly 24 paired primary tasks" + : "end-to-end latency improvement requires success noninferiority and a repeatable CI below zero", + ); + } + + gates.push({ + campaign_id: campaignId, + agent, + success_noninferior: successNoninferior, + initial_schema_tokens_improved: schemaImproved, + end_to_end_latency_improved: latencyImproved, + success_lower_95_percentage_points: successLower, + schema_tokens_upper_95: schemaUpper, + latency_upper_95_ms: latencyUpper, + reasons, + }); + } + } + + const completeMatrix = + campaignCountValid && agents.length > 0 && gates.length === agents.length * campaignIds.length; + return { + required_agents: agents, + required_campaigns: campaignIds, + primary_catalog_size: options.primaryCatalogSize, + noninferiority_margin_percentage_points: margin, + campaign_agent_gates: gates, + cross_agent_success_noninferior: + completeMatrix && gates.every((gate) => gate.success_noninferior), + cross_agent_initial_schema_tokens_improved: + completeMatrix && gates.every((gate) => gate.initial_schema_tokens_improved), + cross_agent_end_to_end_latency_improved: + completeMatrix && gates.every((gate) => gate.end_to_end_latency_improved), + }; +} + +export function buildToolDisclosureSummary( + manifest: ToolDisclosureManifest, + runs: readonly ToolDisclosureRun[], + options: { generatedAt?: string } = {}, +): ToolDisclosureSummary { + validateEvidenceSet(manifest, runs); + const comparisonCells = buildComparisonCells(runs, { + samples: manifest.protocol.bootstrap_samples, + seed: manifest.protocol.bootstrap_seed, + }); + const staticVisibility = summarizeStaticVisibility(runs); + const claimGates = evaluateClaimGates(comparisonCells, staticVisibility, { + agents: manifest.protocol.agents, + campaignIds: manifest.campaigns.map((campaign) => campaign.campaign_id), + primaryCatalogSize: manifest.protocol.primary_catalog_size, + noninferiorityMarginPercentagePoints: manifest.protocol.noninferiority_margin_percentage_points, + }); + const summaryWithoutClaims: ToolDisclosureSummary = { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: manifest.performance_test_id, + generated_at: options.generatedAt ?? new Date().toISOString(), + bootstrap_samples: manifest.protocol.bootstrap_samples, + bootstrap_seed: normalizeSeed(manifest.protocol.bootstrap_seed), + comparison_cells: comparisonCells, + static_visibility: staticVisibility, + claim_gates: claimGates, + claims: [], + }; + return { + ...summaryWithoutClaims, + claims: buildConservativeClaims(summaryWithoutClaims), + }; +} + +/** Public claim text is derived only from gated values; callers cannot supply prose. */ +export function buildConservativeClaims(summary: ToolDisclosureSummary): string[] { + const claims: string[] = []; + const gates = summary.claim_gates; + const targetStaticCells = summary.static_visibility.filter( + (cell) => + gates.required_campaigns.includes(cell.campaign_id) && + gates.required_agents.includes(cell.agent) && + cell.catalog_size === gates.primary_catalog_size, + ); + const expectedCellCount = gates.required_campaigns.length * gates.required_agents.length; + const successMatrixPasses = claimGateMatrixPasses(summary, (gate) => + Boolean( + gate.success_noninferior && + gate.success_lower_95_percentage_points !== undefined && + gate.success_lower_95_percentage_points >= gates.noninferiority_margin_percentage_points, + ), + ); + const schemaMatrixPasses = claimGateMatrixPasses(summary, (gate) => + Boolean( + gate.initial_schema_tokens_improved && + gate.schema_tokens_upper_95 !== undefined && + gate.schema_tokens_upper_95 < 0, + ), + ); + const latencyMatrixPasses = claimGateMatrixPasses(summary, (gate) => + Boolean( + gate.end_to_end_latency_improved && + gate.latency_upper_95_ms !== undefined && + gate.latency_upper_95_ms < 0, + ), + ); + + if ( + gates.cross_agent_initial_schema_tokens_improved && + gates.cross_agent_success_noninferior && + successMatrixPasses && + schemaMatrixPasses && + expectedCellCount > 0 && + targetStaticCells.length === expectedCellCount && + targetStaticCells.every((cell) => cell.reduction.tokenizer_tokens > 0) + ) { + const reductions = targetStaticCells.map((cell) => cell.reduction.tokenizer_tokens_percent); + const minimumReduction = Math.min(...reductions); + const exactNinetyNine = reductions.every((reduction) => reduction >= 99); + const amount = exactNinetyNine + ? "at least 99%" + : `at least ${formatConservativePercent(minimumReduction)}`; + claims.push( + `Across ${gates.required_agents.length} agents and two independent campaigns at ` + + `${gates.primary_catalog_size} catalog tools, progressive disclosure reduced initial ` + + `serialized tool-schema tokens ${amount}.`, + ); + // Defensive invariant: the 99% wording is impossible below the measured threshold. + if (minimumReduction < 99 && claims.at(-1)?.includes("99%")) { + throw new Error("claim generator attempted to overstate schema-token reduction"); + } + } + + if (gates.cross_agent_success_noninferior && successMatrixPasses) { + claims.push( + `Across ${gates.required_agents.length} agents and two independent campaigns at ` + + `${gates.primary_catalog_size} catalog tools, progressive disclosure was noninferior ` + + `to direct exposure for task success at the ${formatNumber( + Math.abs(gates.noninferiority_margin_percentage_points), + )}-percentage-point margin.`, + ); + } + + if ( + gates.cross_agent_end_to_end_latency_improved && + gates.cross_agent_success_noninferior && + successMatrixPasses && + latencyMatrixPasses + ) { + claims.push( + `Across ${gates.required_agents.length} agents and two independent campaigns at ` + + `${gates.primary_catalog_size} catalog tools, progressive disclosure reduced ` + + `end-to-end task latency with paired 95% confidence intervals below zero.`, + ); + } + return claims; +} + +function claimGateMatrixPasses( + summary: ToolDisclosureSummary, + predicate: (gate: CampaignAgentGate) => boolean, +): boolean { + const gates = summary.claim_gates; + if ( + gates.required_campaigns.length !== 2 || + unique(gates.required_campaigns).length !== gates.required_campaigns.length || + gates.required_agents.length === 0 || + unique(gates.required_agents).length !== gates.required_agents.length + ) { + return false; + } + for (const campaignId of gates.required_campaigns) { + for (const agent of gates.required_agents) { + const matching = gates.campaign_agent_gates.filter( + (gate) => gate.campaign_id === campaignId && gate.agent === agent, + ); + if (matching.length !== 1 || !predicate(matching[0])) return false; + } + } + return ( + gates.campaign_agent_gates.length === + gates.required_campaigns.length * gates.required_agents.length + ); +} + +function buildComparisonCell( + group: ComparisonGroup, + samples: number, + baseSeed: number, +): ComparisonCellSummary { + const directRuns = group.runs.filter((run) => run.mode === "direct"); + const progressiveRuns = group.runs.filter((run) => run.mode === "progressive"); + if (directRuns.length === 0 || progressiveRuns.length === 0) { + throw new Error(`comparison requires both modes for ${comparisonGroupLabel(group)}`); + } + const directTasks = aggregateTasks(directRuns); + const progressiveTasks = aggregateTasks(progressiveRuns); + const taskIds = [...directTasks.keys()] + .filter((taskId) => progressiveTasks.has(taskId)) + .sort((left, right) => left.localeCompare(right)); + if (taskIds.length === 0) { + throw new Error(`comparison has no scored task pairs for ${comparisonGroupLabel(group)}`); + } + + const metricSeed = (metric: string): number => + deriveSeed( + baseSeed, + [group.campaignId, group.phase, group.agent, group.catalogSize, metric].join("|"), + ); + const difference = ( + metric: keyof TaskAggregate, + seedName: string, + scale = 1, + ): ConfidenceInterval | undefined => { + const pairs: PairedObservation[] = []; + for (const taskId of taskIds) { + const directValue = directTasks.get(taskId)?.[metric]; + const progressiveValue = progressiveTasks.get(taskId)?.[metric]; + if (directValue === undefined || progressiveValue === undefined) continue; + pairs.push({ direct: directValue * scale, progressive: progressiveValue * scale }); + } + if (pairs.length === 0) return undefined; + return pairedBootstrapDifference(pairs, { samples, seed: metricSeed(seedName) }); + }; + const success = difference("success", "success", 100); + const schema = difference("initialSchemaTokens", "initial-schema-tokens"); + if (!success || !schema) { + throw new Error(`comparison is missing required metrics for ${comparisonGroupLabel(group)}`); + } + + return { + campaign_id: group.campaignId, + phase: group.phase, + agent: group.agent, + catalog_size: group.catalogSize, + direct: aggregateMode(directRuns, directTasks), + progressive: aggregateMode(progressiveRuns, progressiveTasks), + differences: { + success_percentage_points: success, + initial_tool_schema_tokens: schema, + total_prompt_tokens: difference("totalPromptTokens", "total-prompt-tokens"), + time_to_first_response_byte_ms: difference( + "timeToFirstResponseByteMs", + "time-to-first-response-byte", + ), + end_to_end_time_ms: difference("endToEndTimeMs", "end-to-end-time"), + }, + }; +} + +function aggregateTasks(runs: readonly ToolDisclosureRun[]): Map { + const byTask = new Map(); + for (const run of runs) { + if (!run.scored) continue; + const taskRuns = byTask.get(run.task_id) ?? []; + taskRuns.push(run); + byTask.set(run.task_id, taskRuns); + } + return new Map( + [...byTask.entries()].map(([taskId, taskRuns]) => [ + taskId, + { + success: mean(taskRuns.map((run) => Number(run.correctness.task_success))), + initialSchemaTokens: mean( + taskRuns.map((run) => run.measurements.initial_tool_schema.tokenizer_tokens), + ), + totalPromptTokens: optionalMean( + taskRuns.map((run) => run.measurements.total_prompt_tokens), + ), + timeToFirstResponseByteMs: optionalMean( + taskRuns.map((run) => run.measurements.time_to_first_response_byte_ms), + ), + endToEndTimeMs: optionalMean(taskRuns.map((run) => run.measurements.end_to_end_time_ms)), + }, + ]), + ); +} + +function aggregateMode( + runs: readonly ToolDisclosureRun[], + tasks: ReadonlyMap, +): ModeAggregate { + const taskValues = [...tasks.values()]; + return { + runs: runs.length, + scored_runs: runs.filter((run) => run.scored).length, + tasks: tasks.size, + success_rate_percent: round6(mean(taskValues.map((task) => task.success)) * 100), + mean_initial_tool_schema_tokens: round6( + mean(taskValues.map((task) => task.initialSchemaTokens)), + ), + mean_total_prompt_tokens: optionalMean(taskValues.map((task) => task.totalPromptTokens)), + mean_time_to_first_response_byte_ms: optionalMean( + taskValues.map((task) => task.timeToFirstResponseByteMs), + ), + mean_end_to_end_time_ms: optionalMean(taskValues.map((task) => task.endToEndTimeMs)), + }; +} + +function exactVisibilitySnapshot( + group: StaticGroup, + mode: ToolDisclosureMode, +): SchemaVisibilitySnapshot { + const modeRuns = group.runs.filter((run) => run.mode === mode && run.outcome === "success"); + if (modeRuns.length === 0) { + throw new Error(`static visibility is missing ${mode} mode for ${staticGroupLabel(group)}`); + } + const signatures = new Set( + modeRuns.map((run) => { + const value = run.measurements.initial_tool_schema; + return `${value.tool_count}:${value.serialized_bytes}:${value.tokenizer_tokens}`; + }), + ); + if (signatures.size !== 1) { + throw new Error( + `static visibility is nondeterministic for ${mode} mode in ${staticGroupLabel(group)}`, + ); + } + const measurement = modeRuns[0].measurements.initial_tool_schema; + return { ...measurement, samples: modeRuns.length }; +} + +function validateEvidenceSet( + manifest: ToolDisclosureManifest, + runs: readonly ToolDisclosureRun[], +): void { + if (manifest.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION) { + throw new Error(`unsupported manifest schema: ${manifest.schema_version}`); + } + if (manifest.campaigns.length !== 2) { + throw new Error("public claim protocol requires exactly two campaigns"); + } + for (const run of runs) { + if (run.schema_version !== manifest.schema_version) { + throw new Error(`run ${run.run_id} uses a different schema version`); + } + if (run.performance_test_id !== manifest.performance_test_id) { + throw new Error(`run ${run.run_id} belongs to a different performance test`); + } + } +} + +function validateRun(run: ToolDisclosureRun): void { + if (run.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION) { + throw new Error(`run ${run.run_id} has unsupported schema ${run.schema_version}`); + } + const shouldBeScored = run.phase !== "static-visibility" && run.outcome !== "setup-error"; + if (run.phase !== "static-visibility" && !run.task_kind) { + throw new Error(`run ${run.run_id} is missing its task kind`); + } + if (run.scored !== shouldBeScored) { + throw new Error(`run ${run.run_id} has inconsistent scored/outcome fields`); + } + if ( + run.phase !== "static-visibility" && + run.correctness.task_success !== (run.outcome === "success") + ) { + throw new Error(`run ${run.run_id} has inconsistent success/outcome fields`); + } + for (const [name, value] of Object.entries(run.measurements.initial_tool_schema)) { + assertNonnegativeInteger(value, `${run.run_id} initial_tool_schema.${name}`); + } +} + +function compareComparisonGroups(left: ComparisonGroup, right: ComparisonGroup): number { + return ( + left.campaignId.localeCompare(right.campaignId) || + PHASE_ORDER[left.phase] - PHASE_ORDER[right.phase] || + left.agent.localeCompare(right.agent) || + left.catalogSize - right.catalogSize + ); +} + +function compareStaticGroups(left: StaticGroup, right: StaticGroup): number { + return ( + left.campaignId.localeCompare(right.campaignId) || + left.agent.localeCompare(right.agent) || + left.catalogSize - right.catalogSize + ); +} + +function comparisonGroupLabel(group: ComparisonGroup): string { + return `${group.campaignId}/${group.phase}/${group.agent}/${group.catalogSize}`; +} + +function staticGroupLabel(group: StaticGroup): string { + return `${group.campaignId}/${group.agent}/${group.catalogSize}`; +} + +function optionalMean(values: readonly (number | undefined)[]): number | undefined { + const present = values.filter((value): value is number => value !== undefined); + if (present.length === 0) return undefined; + present.forEach((value) => assertFinite(value, "metric value")); + return round6(mean(present)); +} + +function mean(values: readonly number[]): number { + if (values.length === 0) throw new Error("mean requires at least one value"); + return values.reduce((total, value) => total + value, 0) / values.length; +} + +function quantile(sortedValues: readonly number[], probability: number): number { + const position = (sortedValues.length - 1) * probability; + const lowerIndex = Math.floor(position); + const upperIndex = Math.ceil(position); + const weight = position - lowerIndex; + return sortedValues[lowerIndex] * (1 - weight) + sortedValues[upperIndex] * weight; +} + +function deriveSeed(baseSeed: number, label: string): number { + let hash = normalizeSeed(baseSeed) ^ 0x81_1c_9d_c5; + for (let index = 0; index < label.length; index += 1) { + hash ^= label.charCodeAt(index); + hash = Math.imul(hash, 0x01_00_01_93); + } + return normalizeSeed(hash); +} + +function normalizeSeed(seed: number): number { + if (!Number.isInteger(seed)) + throw new Error(`bootstrap seed must be an integer, received ${seed}`); + return seed >>> 0; +} + +function mulberry32(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d_2b_79_f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296; + }; +} + +function unique(values: readonly T[]): T[] { + return [...new Set(values)]; +} + +function round6(value: number): number { + return Number(value.toFixed(6)); +} + +function formatNumber(value: number | undefined): string { + if (value === undefined) return "unavailable"; + return Number.isInteger(value) + ? String(value) + : value.toFixed(2).replace(/0+$/, "").replace(/\.$/, ""); +} + +function formatConservativePercent(value: number): string { + // One decimal avoids rounding a sub-99 result into the public "99%" headline. + const floored = Math.floor(value * 10) / 10; + return `${formatNumber(floored)}%`; +} + +function assertFinite(value: number, label: string): void { + if (!Number.isFinite(value)) throw new Error(`${label} must be finite`); +} + +function assertNonnegativeInteger(value: number, label: string): void { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${label} must be a nonnegative integer`); + } +} diff --git a/scripts/performance/tool-disclosure/subprocess.ts b/scripts/performance/tool-disclosure/subprocess.ts new file mode 100644 index 0000000000..c2bc1c3978 --- /dev/null +++ b/scripts/performance/tool-disclosure/subprocess.ts @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn } from "node:child_process"; +import { performance } from "node:perf_hooks"; + +export interface SubprocessResult { + exit_code: number | null; + signal: NodeJS.Signals | null; + elapsed_ms: number; + stdout: string; + stderr: string; + timed_out: boolean; + output_truncated: boolean; +} + +function scrub(text: string, redactions: readonly string[]): string { + let result = text; + for (const value of redactions) { + if (value) result = result.replaceAll(value, ""); + } + return result; +} + +export async function runBoundedCommand(options: { + command: string; + args: readonly string[]; + cwd?: string; + env?: NodeJS.ProcessEnv; + timeoutMs: number; + maxOutputBytes?: number; + redactions?: readonly string[]; +}): Promise { + if (!options.command || options.command.includes("\0")) + throw new Error("invalid performance test command"); + if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1) { + throw new Error("performance test command timeout must be a positive integer"); + } + const maxOutputBytes = options.maxOutputBytes ?? 2 * 1024 * 1024; + const child = spawn(options.command, [...options.args], { + cwd: options.cwd, + env: options.env, + detached: process.platform !== "win32", + stdio: ["ignore", "pipe", "pipe"], + }); + const startedAt = performance.now(); + let stdout: Buffer = Buffer.alloc(0); + let stderr: Buffer = Buffer.alloc(0); + let truncated = false; + const append = ( + current: Buffer, + chunk: Buffer, + ): Buffer => { + if (current.length >= maxOutputBytes) { + truncated = true; + return current; + } + const remaining = maxOutputBytes - current.length; + if (chunk.length > remaining) truncated = true; + return Buffer.concat([current, chunk.subarray(0, remaining)]); + }; + child.stdout.on("data", (chunk: Buffer) => { + stdout = append(stdout, chunk); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr = append(stderr, chunk); + }); + + let timedOut = false; + let hardStop: NodeJS.Timeout | undefined; + const timer = setTimeout(() => { + timedOut = true; + try { + if (child.pid && process.platform !== "win32") process.kill(-child.pid, "SIGTERM"); + else child.kill("SIGTERM"); + } catch { + // Process already exited. + } + hardStop = setTimeout(() => { + try { + if (child.pid && process.platform !== "win32") process.kill(-child.pid, "SIGKILL"); + else child.kill("SIGKILL"); + } catch { + // Process already exited. + } + }, 5_000); + hardStop.unref(); + }, options.timeoutMs); + + const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal })); + }, + ).finally(() => { + clearTimeout(timer); + if (hardStop) clearTimeout(hardStop); + }); + const redactions = options.redactions ?? []; + return { + exit_code: result.code, + signal: result.signal, + elapsed_ms: Number((performance.now() - startedAt).toFixed(3)), + stdout: scrub(stdout.toString("utf8"), redactions), + stderr: scrub(stderr.toString("utf8"), redactions), + timed_out: timedOut, + output_truncated: truncated, + }; +} diff --git a/scripts/performance/tool-disclosure/tasks.ts b/scripts/performance/tool-disclosure/tasks.ts new file mode 100644 index 0000000000..97d6b7238c --- /dev/null +++ b/scripts/performance/tool-disclosure/tasks.ts @@ -0,0 +1,369 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + buildSyntheticArguments, + canonicalJson, + executeSyntheticTool, + type JsonObject, + type JsonValue, + SYNTHETIC_CATEGORIES, + type SyntheticCatalog, + type SyntheticTool, + sha256Hex, + validateSyntheticArguments, +} from "./catalog"; + +export const TOOL_DISCLOSURE_PERFORMANCE_TEST_TASK_SCHEMA_VERSION = + "nemoclaw.tool_disclosure.performance_test.tasks.v1" as const; +export const PRIMARY_TASK_COUNT = 24; +export const PRIMARY_CATALOG_MIN_SIZE = 64; +export const STRESS_TASK_COUNT = 8; +export const STRESS_CATALOG_SIZE = 2_209; + +export type PerformanceTestTaskKind = "single-tool" | "ordered-chain" | "near-match" | "no-tool"; +export type PerformanceTestTaskSuite = "primary" | "stress"; + +export interface ExpectedToolCall { + tool_name: string; + arguments: JsonObject; + result_nonce: string; +} + +export interface SyntheticPerformanceTask { + id: string; + suite: PerformanceTestTaskSuite; + kind: PerformanceTestTaskKind; + prompt: string; + min_catalog_size: number; + expected_calls: ExpectedToolCall[]; + expected_final_includes: string[]; + distractor_tool_names: string[]; +} + +export interface SyntheticTaskSet { + schema_version: typeof TOOL_DISCLOSURE_PERFORMANCE_TEST_TASK_SCHEMA_VERSION; + catalog_seed: string; + suite: PerformanceTestTaskSuite; + task_count: number; + tasks_sha256: string; + tasks: SyntheticPerformanceTask[]; +} + +function toolAt(catalog: SyntheticCatalog, index: number): SyntheticTool { + const tool = catalog.tools[index]; + if (!tool) throw new RangeError(`catalog does not contain required tool index ${index}`); + return tool; +} + +function categoryDetails(tool: SyntheticTool): (typeof SYNTHETIC_CATEGORIES)[number] { + const category = SYNTHETIC_CATEGORIES.find((candidate) => candidate.id === tool.category); + if (!category) throw new Error(`unknown synthetic category ${tool.category}`); + return category; +} + +function expectedCall(tool: SyntheticTool, args: JsonObject): ExpectedToolCall { + return { + tool_name: tool.definition.function.name, + arguments: args, + result_nonce: executeSyntheticTool(tool, args).nonce, + }; +} + +function operationText(tool: SyntheticTool): string { + const category = categoryDetails(tool); + return `the ${tool.operation.replaceAll("_", " ")} operation for a synthetic ${category.resource} in ${category.label}`; +} + +function positiveFinalInstruction(callCount: number): string { + return callCount === 1 + ? "Include the returned nonce exactly in the final answer." + : "Include both returned nonces exactly, in call order, in the final answer."; +} + +function buildSingleTask( + suite: PerformanceTestTaskSuite, + ordinal: number, + tool: SyntheticTool, + variant: number, +): SyntheticPerformanceTask { + const args = buildSyntheticArguments(tool, variant); + const call = expectedCall(tool, args); + return { + id: `${suite}-single-${String(ordinal).padStart(2, "0")}`, + suite, + kind: "single-tool", + prompt: [ + `Use the available tools to perform ${operationText(tool)}.`, + `Pass exactly these arguments: ${canonicalJson(args)}.`, + positiveFinalInstruction(1), + ].join(" "), + min_catalog_size: tool.index + 1, + expected_calls: [call], + expected_final_includes: [call.result_nonce], + distractor_tool_names: [], + }; +} + +function buildChainTask( + ordinal: number, + firstTool: SyntheticTool, + secondTool: SyntheticTool, +): SyntheticPerformanceTask { + const firstArgs = buildSyntheticArguments(firstTool, 200 + ordinal); + const firstCall = expectedCall(firstTool, firstArgs); + const secondArgs = buildSyntheticArguments(secondTool, 300 + ordinal, { + resource_id: firstCall.result_nonce, + }); + const secondCall = expectedCall(secondTool, secondArgs); + const secondFixedArgs = { ...secondArgs }; + delete secondFixedArgs.resource_id; + + return { + id: `primary-chain-${String(ordinal).padStart(2, "0")}`, + suite: "primary", + kind: "ordered-chain", + prompt: [ + `Complete two tool operations in order. First, perform ${operationText(firstTool)}`, + `with exactly these arguments: ${canonicalJson(firstArgs)}.`, + `Then perform ${operationText(secondTool)}. For the second call, set resource_id to the nonce`, + `returned by the first call and pass these other fields exactly: ${canonicalJson(secondFixedArgs)}.`, + positiveFinalInstruction(2), + ].join(" "), + min_catalog_size: Math.max(firstTool.index, secondTool.index) + 1, + expected_calls: [firstCall, secondCall], + expected_final_includes: [firstCall.result_nonce, secondCall.result_nonce], + distractor_tool_names: [], + }; +} + +function buildNearMatchTask( + ordinal: number, + target: SyntheticTool, + distractors: readonly SyntheticTool[], +): SyntheticPerformanceTask { + const args = buildSyntheticArguments(target, 400 + ordinal); + const call = expectedCall(target, args); + const distractorOperations = distractors + .map((tool) => tool.operation.replaceAll("_", " ")) + .join(" or "); + + return { + id: `primary-near-match-${String(ordinal).padStart(2, "0")}`, + suite: "primary", + kind: "near-match", + prompt: [ + `Use exactly one tool to perform ${operationText(target)}.`, + `Do not perform the nearby ${distractorOperations} operations in the same category.`, + `Pass exactly these arguments: ${canonicalJson(args)}.`, + positiveFinalInstruction(1), + ].join(" "), + min_catalog_size: Math.max(target.index, ...distractors.map((tool) => tool.index)) + 1, + expected_calls: [call], + expected_final_includes: [call.result_nonce], + distractor_tool_names: distractors.map((tool) => tool.definition.function.name), + }; +} + +const NO_TOOL_CONTROLS = [ + { + phrase: "CONTROL-NO-TOOL-ALPHA", + prompt: "Without using tools, reply with the exact phrase CONTROL-NO-TOOL-ALPHA.", + }, + { + phrase: "CONTROL-NO-TOOL-BRAVO", + prompt: "Do not call a tool. Return only the exact phrase CONTROL-NO-TOOL-BRAVO.", + }, + { + phrase: "CONTROL-NO-TOOL-CHARLIE", + prompt: "This is a no-tool control: answer with exactly CONTROL-NO-TOOL-CHARLIE.", + }, + { + phrase: "CONTROL-NO-TOOL-DELTA", + prompt: "Use no external capability and respond with exactly CONTROL-NO-TOOL-DELTA.", + }, +] as const; + +function buildNoToolTasks(): SyntheticPerformanceTask[] { + return NO_TOOL_CONTROLS.map((control, index) => ({ + id: `primary-no-tool-${String(index + 1).padStart(2, "0")}`, + suite: "primary", + kind: "no-tool", + prompt: control.prompt, + min_catalog_size: 1, + expected_calls: [], + expected_final_includes: [control.phrase], + distractor_tool_names: [], + })); +} + +export function buildPrimaryTasks(catalog: SyntheticCatalog): SyntheticPerformanceTask[] { + if (catalog.size < PRIMARY_CATALOG_MIN_SIZE) { + throw new RangeError( + `primary tasks require at least ${PRIMARY_CATALOG_MIN_SIZE} tools; catalog has ${catalog.size}`, + ); + } + + const singleTasks = Array.from({ length: 8 }, (_, index) => + buildSingleTask("primary", index + 1, toolAt(catalog, index), 100 + index), + ); + const chainTasks = Array.from({ length: 8 }, (_, index) => + buildChainTask(index + 1, toolAt(catalog, 8 + index * 2), toolAt(catalog, 9 + index * 2)), + ); + const nearMatchTasks = Array.from({ length: 4 }, (_, index) => + buildNearMatchTask(index + 1, toolAt(catalog, 24 + index), [ + toolAt(catalog, index), + toolAt(catalog, 48 + index), + ]), + ); + return [...singleTasks, ...chainTasks, ...nearMatchTasks, ...buildNoToolTasks()]; +} + +const STRESS_TOOL_INDICES = [256, 511, 767, 1_023, 1_279, 1_535, 1_791, 2_208] as const; + +export function buildStressTasks(catalog: SyntheticCatalog): SyntheticPerformanceTask[] { + if (catalog.size < STRESS_CATALOG_SIZE) { + throw new RangeError( + `stress tasks require ${STRESS_CATALOG_SIZE} tools; catalog has ${catalog.size}`, + ); + } + return STRESS_TOOL_INDICES.map((toolIndex, index) => + buildSingleTask("stress", index + 1, toolAt(catalog, toolIndex), 500 + index), + ); +} + +function buildTaskSet( + catalog: SyntheticCatalog, + suite: PerformanceTestTaskSuite, + tasks: SyntheticPerformanceTask[], +): SyntheticTaskSet { + return { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_TASK_SCHEMA_VERSION, + catalog_seed: catalog.seed, + suite, + task_count: tasks.length, + tasks_sha256: sha256Hex(canonicalJson(tasks as unknown as JsonValue)), + tasks, + }; +} + +export function generatePrimaryTaskSet(catalog: SyntheticCatalog): SyntheticTaskSet { + return buildTaskSet(catalog, "primary", buildPrimaryTasks(catalog)); +} + +export function generateStressTaskSet(catalog: SyntheticCatalog): SyntheticTaskSet { + return buildTaskSet(catalog, "stress", buildStressTasks(catalog)); +} + +function expectedKindCounts( + suite: PerformanceTestTaskSuite, +): Readonly> { + return suite === "primary" + ? { "single-tool": 8, "ordered-chain": 8, "near-match": 4, "no-tool": 4 } + : { "single-tool": 8, "ordered-chain": 0, "near-match": 0, "no-tool": 0 }; +} + +export function validateSyntheticTaskSet( + taskSet: SyntheticTaskSet, + catalog: SyntheticCatalog, +): string[] { + const errors: string[] = []; + if (taskSet.schema_version !== TOOL_DISCLOSURE_PERFORMANCE_TEST_TASK_SCHEMA_VERSION) { + errors.push(`unsupported task schema version: ${taskSet.schema_version}`); + } + if (taskSet.catalog_seed !== catalog.seed) errors.push("task and catalog seeds do not match"); + if (taskSet.task_count !== taskSet.tasks.length) errors.push("task_count does not match tasks"); + const expectedHash = sha256Hex(canonicalJson(taskSet.tasks as unknown as JsonValue)); + if (taskSet.tasks_sha256 !== expectedHash) errors.push("tasks_sha256 does not match tasks"); + + const ids = new Set(); + const tools = new Map(catalog.tools.map((tool) => [tool.definition.function.name, tool])); + const counts: Record = { + "single-tool": 0, + "ordered-chain": 0, + "near-match": 0, + "no-tool": 0, + }; + + for (const task of taskSet.tasks) { + counts[task.kind] += 1; + if (ids.has(task.id)) errors.push(`duplicate task id: ${task.id}`); + ids.add(task.id); + if (task.suite !== taskSet.suite) errors.push(`${task.id} has the wrong suite`); + if (task.prompt.trim().length === 0) errors.push(`${task.id} has an empty prompt`); + if (task.min_catalog_size > catalog.size) errors.push(`${task.id} exceeds catalog size`); + + for (const call of task.expected_calls) { + const tool = tools.get(call.tool_name); + if (!tool) { + errors.push(`${task.id} references missing tool ${call.tool_name}`); + continue; + } + const argumentErrors = validateSyntheticArguments(tool, call.arguments); + if (argumentErrors.length > 0) { + errors.push(`${task.id} has invalid arguments for ${call.tool_name}`); + continue; + } + const result = executeSyntheticTool(tool, call.arguments); + if (result.nonce !== call.result_nonce) + errors.push(`${task.id} has an incorrect result nonce`); + if (tool.index + 1 > task.min_catalog_size) + errors.push(`${task.id} has a low min_catalog_size`); + if (!task.expected_final_includes.includes(call.result_nonce)) { + errors.push(`${task.id} does not require its result nonce in the final answer`); + } + } + + if (task.kind === "single-tool" && task.expected_calls.length !== 1) { + errors.push(`${task.id} must expect exactly one call`); + } + if (task.kind === "ordered-chain") { + if (task.expected_calls.length !== 2) { + errors.push(`${task.id} must expect exactly two calls`); + } else if ( + task.expected_calls[1].arguments.resource_id !== task.expected_calls[0].result_nonce + ) { + errors.push(`${task.id} does not pass the first nonce into the second call`); + } + } + if (task.kind === "near-match") { + if (task.expected_calls.length !== 1 || task.distractor_tool_names.length < 1) { + errors.push(`${task.id} must have one expected call and at least one distractor`); + } + for (const distractor of task.distractor_tool_names) { + if (!tools.has(distractor)) + errors.push(`${task.id} references missing distractor ${distractor}`); + if (task.expected_calls.some((call) => call.tool_name === distractor)) { + errors.push(`${task.id} calls a declared distractor`); + } + } + } + if (task.kind === "no-tool" && task.expected_calls.length !== 0) { + errors.push(`${task.id} must not expect tool calls`); + } + } + + const expectedCounts = expectedKindCounts(taskSet.suite); + for (const kind of Object.keys(counts) as PerformanceTestTaskKind[]) { + if (counts[kind] !== expectedCounts[kind]) { + errors.push( + `${taskSet.suite} task set has ${counts[kind]} ${kind} tasks; expected ${expectedCounts[kind]}`, + ); + } + } + const expectedTotal = taskSet.suite === "primary" ? PRIMARY_TASK_COUNT : STRESS_TASK_COUNT; + if (taskSet.tasks.length !== expectedTotal) { + errors.push( + `${taskSet.suite} task set has ${taskSet.tasks.length} tasks; expected ${expectedTotal}`, + ); + } + return errors; +} + +export function assertValidSyntheticTaskSet( + taskSet: SyntheticTaskSet, + catalog: SyntheticCatalog, +): void { + const errors = validateSyntheticTaskSet(taskSet, catalog); + if (errors.length > 0) throw new Error(`invalid synthetic task set: ${errors.join("; ")}`); +} diff --git a/scripts/performance/tool-disclosure/telemetry.ts b/scripts/performance/tool-disclosure/telemetry.ts new file mode 100644 index 0000000000..7085b84c11 --- /dev/null +++ b/scripts/performance/tool-disclosure/telemetry.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isIP } from "node:net"; + +const METRIC_NAMES = { + prompt: ["vllm:prompt_tokens_total", "vllm_prompt_tokens_total"], + generation: ["vllm:generation_tokens_total", "vllm_generation_tokens_total"], +} as const; + +export interface VllmTokenSnapshot { + prompt_tokens: number; + generation_tokens: number; +} + +export interface VllmTokenDelta extends VllmTokenSnapshot { + available: boolean; +} + +function isLoopback(hostname: string): boolean { + const normalized = hostname.replace(/^\[/u, "").replace(/\]$/u, "").toLowerCase(); + return ( + normalized === "localhost" || + normalized === "::1" || + (isIP(normalized) === 4 && normalized.startsWith("127.")) + ); +} + +export function validatedTelemetryUrl(raw: string, endpoint: string): URL { + let url: URL; + try { + url = new URL(raw); + } catch { + throw new Error("vLLM telemetry URL must be a valid HTTP(S) URL"); + } + if (!new Set(["http:", "https:"]).has(url.protocol)) { + throw new Error("vLLM telemetry URL must use HTTP or HTTPS"); + } + if (url.username || url.password) { + throw new Error("vLLM telemetry URL must not contain credentials"); + } + if (!isLoopback(url.hostname)) { + throw new Error("vLLM telemetry is allowed only on loopback"); + } + if (url.hostname.toLowerCase() === "localhost") url.hostname = "127.0.0.1"; + url.username = ""; + url.password = ""; + url.search = ""; + url.hash = ""; + url.pathname = endpoint; + return url; +} + +function metricFamilyValue(text: string, names: readonly string[]): number | undefined { + let total = 0; + let found = false; + for (const line of text.split(/\r?\n/u)) { + if (!line || line.startsWith("#")) continue; + const separator = line.lastIndexOf(" "); + if (separator < 1) continue; + const identity = line.slice(0, separator); + const name = identity.replace(/\{.*$/u, ""); + if (!names.includes(name)) continue; + const value = Number(line.slice(separator + 1)); + if (!Number.isFinite(value) || value < 0) continue; + total += value; + found = true; + } + return found ? total : undefined; +} + +export function parseVllmTokenMetrics(text: string): VllmTokenSnapshot | null { + const prompt = metricFamilyValue(text, METRIC_NAMES.prompt); + const generation = metricFamilyValue(text, METRIC_NAMES.generation); + if (prompt === undefined || generation === undefined) return null; + return { prompt_tokens: prompt, generation_tokens: generation }; +} + +export function parseProcessStartTime(text: string): number | null { + return metricFamilyValue(text, ["process_start_time_seconds"]) ?? null; +} + +export async function readVllmProcessStartTime( + baseUrl: string, + fetchImpl: typeof fetch = fetch, + timeoutMs = 5_000, +): Promise { + const response = await fetchImpl(validatedTelemetryUrl(baseUrl, "/metrics"), { + method: "GET", + redirect: "error", + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`vLLM metrics request failed with HTTP ${response.status}`); + } + const started = parseProcessStartTime(await response.text()); + if (started === null) throw new Error("vLLM metrics did not expose process_start_time_seconds"); + return started; +} + +export async function readVllmTokenSnapshot( + baseUrl: string, + fetchImpl: typeof fetch = fetch, + timeoutMs = 5_000, +): Promise { + const response = await fetchImpl(validatedTelemetryUrl(baseUrl, "/metrics"), { + method: "GET", + redirect: "error", + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + await response.body?.cancel(); + return null; + } + return parseVllmTokenMetrics(await response.text()); +} + +export function tokenDelta( + before: VllmTokenSnapshot | null, + after: VllmTokenSnapshot | null, +): VllmTokenDelta { + if (!before || !after) return { prompt_tokens: 0, generation_tokens: 0, available: false }; + const prompt = after.prompt_tokens - before.prompt_tokens; + const generation = after.generation_tokens - before.generation_tokens; + if (prompt < 0 || generation < 0) { + return { prompt_tokens: 0, generation_tokens: 0, available: false }; + } + return { prompt_tokens: prompt, generation_tokens: generation, available: true }; +} + +function tokenCountFromResponse(payload: unknown): number | null { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null; + const record = payload as Record; + for (const key of ["count", "token_count", "num_tokens"]) { + const value = record[key]; + if (Number.isSafeInteger(value) && Number(value) >= 0) return Number(value); + } + const tokens = record.tokens; + return Array.isArray(tokens) ? tokens.length : null; +} + +export async function countTokensWithVllm( + baseUrl: string, + model: string, + text: string, + fetchImpl: typeof fetch = fetch, + timeoutMs = 30_000, +): Promise { + if (!model.trim()) throw new Error("tokenizer model must not be empty"); + const response = await fetchImpl(validatedTelemetryUrl(baseUrl, "/tokenize"), { + method: "POST", + redirect: "error", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model, prompt: text }), + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`vLLM tokenizer request failed with HTTP ${response.status}`); + } + const payload = (await response.json()) as unknown; + const count = tokenCountFromResponse(payload); + if (count === null) throw new Error("vLLM tokenizer response did not contain a token count"); + return count; +} diff --git a/scripts/performance/tool-disclosure/types.ts b/scripts/performance/tool-disclosure/types.ts new file mode 100644 index 0000000000..4dca7fa51a --- /dev/null +++ b/scripts/performance/tool-disclosure/types.ts @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Public evidence schema for the progressive tool-disclosure performance test. + * + * The schema deliberately stores task identifiers, classifications, booleans, + * and numeric measurements rather than prompts, tool arguments, response + * bodies, endpoint URLs, or credentials. Sanitized transcripts are separate, + * checksummed artifacts and never flow through the statistics/report layer. + */ + +export const TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION = + "nemoclaw.tool_disclosure_performance_test.v1" as const; +export const DEFAULT_BOOTSTRAP_SAMPLES = 10_000; +export const DEFAULT_BOOTSTRAP_SEED = 0x4e_56_44_41; +export const DEFAULT_NONINFERIORITY_MARGIN_PP = -5; +export const NO_ACCELERATOR_TYPE = "none" as const; +export const NOT_APPLICABLE = "not-applicable" as const; + +export type ToolDisclosurePerformanceTestSchemaVersion = + typeof TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION; +export type ToolDisclosureAgent = "langchain-deepagents-code" | "hermes" | "openclaw"; +export type ToolDisclosureMode = "direct" | "progressive"; +export type PerformanceTestPhase = + | "static-visibility" + | "small-control" + | "primary" + | "large-stress"; +export type PerformanceTestTaskKind = "single-tool" | "ordered-chain" | "near-match" | "no-tool"; +export type PerformanceTestRunOutcome = + | "success" + | "incorrect" + | "timeout" + | "model-error" + | "context-overflow" + | "setup-error"; + +export interface RevisionIdentity { + git_sha: string; + git_ref: string; + worktree_clean: boolean; +} + +export interface CampaignDefinition { + campaign_id: string; + ordinal: 1 | 2; + fresh_inference_process: boolean; + fresh_sandboxes: boolean; +} + +/** Public, sanitized machine metadata. There is intentionally no hostname or path field. */ +export interface PerformanceTestEnvironment { + operating_system: string; + architecture: string; + cpu_model: string; + cpu_count: number; + ram_gib: number; + /** Generic accelerator metadata. Use count 0, type "none", and explicit not-applicable values for CPU-only runs. */ + accelerator_type: string; + accelerator_model: string; + accelerator_architecture: string; + accelerator_count: number; + accelerator_driver_version: string; + accelerator_runtime: string; + power_state: string; + openshell_version: string; + agent_versions: Record; + /** Exact OCI image digest used by every agent/mode/catalog sandbox cell. */ + sandbox_image_digests?: Readonly>; +} + +export interface InferenceConfiguration { + api: "chat-completions"; + model_id: string; + model_revision: string; + container_image: string; + container_digest: string; + vllm_version: string; + tool_call_parser: string; + reasoning_parser: string; + temperature: 0; + concurrency: 1; + prefix_caching_enabled: boolean; + /** Public, reviewed flags only. Never put endpoint, token, mount, or host values here. */ + public_vllm_flags: readonly string[]; +} + +export interface PerformanceTestProtocol { + agents: readonly ToolDisclosureAgent[]; + modes: readonly ToolDisclosureMode[]; + catalog_sizes: readonly number[]; + primary_catalog_size: number; + tasks: readonly { + task_id: string; + kind: PerformanceTestTaskKind; + }[]; + repetitions: Readonly, number>>; + execution_seed: number; + bootstrap_samples: number; + bootstrap_seed: number; + noninferiority_margin_percentage_points: number; + retry_setup_failures: number; +} + +export interface ToolDisclosureManifest { + schema_version: ToolDisclosurePerformanceTestSchemaVersion; + performance_test_id: string; + created_at: string; + sut: RevisionIdentity; + harness: RevisionIdentity; + campaigns: readonly CampaignDefinition[]; + protocol: PerformanceTestProtocol; + environment: PerformanceTestEnvironment; + inference: InferenceConfiguration; +} + +export interface InitialToolSchemaMeasurement { + tool_count: number; + serialized_bytes: number; + tokenizer_tokens: number; +} + +export interface RunCorrectness { + task_success: boolean; + expected_tool_names: boolean; + expected_tool_order: boolean; + expected_arguments: boolean; + expected_call_count: boolean; + nonce_present: boolean; + unnecessary_tool_calls: number; +} + +export interface RunMeasurements { + initial_tool_schema: InitialToolSchemaMeasurement; + total_prompt_tokens?: number; + completion_tokens?: number; + time_to_first_response_byte_ms?: number; + inference_time_ms?: number; + end_to_end_time_ms?: number; + model_calls: number; + discovery_calls: number; +} + +export interface ToolDisclosureRun { + schema_version: ToolDisclosurePerformanceTestSchemaVersion; + performance_test_id: string; + campaign_id: string; + run_id: string; + phase: PerformanceTestPhase; + agent: ToolDisclosureAgent; + mode: ToolDisclosureMode; + catalog_size: number; + task_id: string; + task_kind?: PerformanceTestTaskKind; + repetition: number; + execution_seed: number; + outcome: PerformanceTestRunOutcome; + /** Static captures are unscored; for task phases, only setup-error is excluded. */ + scored: boolean; + correctness: RunCorrectness; + measurements: RunMeasurements; +} + +export interface ConfidenceInterval { + estimate: number; + lower_95: number; + upper_95: number; + paired_tasks: number; + bootstrap_samples: number; + bootstrap_seed: number; +} + +export interface ModeAggregate { + runs: number; + scored_runs: number; + tasks: number; + success_rate_percent: number; + mean_initial_tool_schema_tokens: number; + mean_total_prompt_tokens?: number; + mean_time_to_first_response_byte_ms?: number; + mean_end_to_end_time_ms?: number; +} + +export interface ComparisonDifferences { + /** Progressive minus direct, in percentage points. Higher is better. */ + success_percentage_points: ConfidenceInterval; + /** Progressive minus direct. Negative is better for all remaining metrics. */ + initial_tool_schema_tokens: ConfidenceInterval; + total_prompt_tokens?: ConfidenceInterval; + time_to_first_response_byte_ms?: ConfidenceInterval; + end_to_end_time_ms?: ConfidenceInterval; +} + +export interface ComparisonCellSummary { + campaign_id: string; + phase: Exclude; + agent: ToolDisclosureAgent; + catalog_size: number; + direct: ModeAggregate; + progressive: ModeAggregate; + differences: ComparisonDifferences; +} + +export interface SchemaVisibilitySnapshot extends InitialToolSchemaMeasurement { + samples: number; +} + +export interface SchemaReductionSummary { + campaign_id: string; + agent: ToolDisclosureAgent; + catalog_size: number; + direct: SchemaVisibilitySnapshot; + progressive: SchemaVisibilitySnapshot; + reduction: { + tool_count: number; + serialized_bytes: number; + tokenizer_tokens: number; + tokenizer_tokens_percent: number; + }; +} + +export interface CampaignAgentGate { + campaign_id: string; + agent: ToolDisclosureAgent; + success_noninferior: boolean; + initial_schema_tokens_improved: boolean; + end_to_end_latency_improved: boolean; + success_lower_95_percentage_points?: number; + schema_tokens_upper_95?: number; + latency_upper_95_ms?: number; + reasons: readonly string[]; +} + +export interface ClaimGateSummary { + required_agents: readonly ToolDisclosureAgent[]; + required_campaigns: readonly string[]; + primary_catalog_size: number; + noninferiority_margin_percentage_points: number; + campaign_agent_gates: readonly CampaignAgentGate[]; + cross_agent_success_noninferior: boolean; + cross_agent_initial_schema_tokens_improved: boolean; + cross_agent_end_to_end_latency_improved: boolean; +} + +export interface ToolDisclosureSummary { + schema_version: ToolDisclosurePerformanceTestSchemaVersion; + performance_test_id: string; + generated_at: string; + bootstrap_samples: number; + bootstrap_seed: number; + comparison_cells: readonly ComparisonCellSummary[]; + static_visibility: readonly SchemaReductionSummary[]; + claim_gates: ClaimGateSummary; + /** Mechanically generated, public-safe claims. Empty means no claim cleared its gate. */ + claims: readonly string[]; +} + +export type EvidenceArtifactKind = + | "catalog" + | "tasks" + | "schedule" + | "fixture-manifest" + | "manifest" + | "runs" + | "summary" + | "report" + | "sanitized-transcript" + | "raw-events" + | "attempt-journal" + | "attestation" + | "checksums"; + +export interface EvidenceArtifact { + artifact_id: string; + kind: EvidenceArtifactKind; + /** A public-safe leaf name, not an absolute path. */ + file_name: string; + media_type: string; + byte_length: number; + sha256: string; + campaign_id?: string; + run_id?: string; +} + +export interface EvidenceBundle { + schema_version: ToolDisclosurePerformanceTestSchemaVersion; + performance_test_id: string; + generated_at: string; + artifacts: readonly EvidenceArtifact[]; +} diff --git a/scripts/performance/tool-disclosure/validation.ts b/scripts/performance/tool-disclosure/validation.ts new file mode 100644 index 0000000000..e4a769d6dd --- /dev/null +++ b/scripts/performance/tool-disclosure/validation.ts @@ -0,0 +1,450 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { assembleToolDisclosureRun } from "./assemble-run"; +import type { SyntheticCatalog } from "./catalog"; +import type { SanitizedRunEvidence } from "./execute"; +import { + buildToolDisclosureSchedule, + type ScheduledToolDisclosureRun, + STATIC_CATALOG_SIZES, + TOOL_DISCLOSURE_AGENTS, + TOOL_DISCLOSURE_MODES, +} from "./schedule"; +import type { SyntheticTaskSet } from "./tasks"; +import type { ToolDisclosureManifest, ToolDisclosureRun } from "./types"; +import { + DEFAULT_BOOTSTRAP_SAMPLES, + DEFAULT_BOOTSTRAP_SEED, + DEFAULT_NONINFERIORITY_MARGIN_PP, + NO_ACCELERATOR_TYPE, + NOT_APPLICABLE, +} from "./types"; + +function sameJson(left: unknown, right: unknown): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +const SECRET_LIKE_COMPONENT = + /(?:^|[^a-z0-9])(?:api[-_ ]?keys?|tokens?|secrets?|passwords?|authorizations?)(?:$|[^a-z0-9])/iu; +const SECRET_LIKE_VALUE = /\b(?:bearer\s+\S+|(?:github_pat|ghp|gho|nvapi|sk)-[a-z0-9_-]{8,})\b/iu; +const URL_LIKE_VALUE = /\b[a-z][a-z0-9+.-]*:\/\/\S*/iu; +const ABSOLUTE_HOST_PATH = + /(?:^|[=\s"'(:,])(?:\/[a-z0-9._~-][^\s,;]*|[a-z]:[\\/][^\s,;]*|\\\\[^\\\s]+\\[^\s,;]*)/iu; +const PRIVATE_ROUTING_FLAGS = new Set([ + "--admin-key", + "--api-key", + "--base-url", + "--endpoint", + "--host", + "--mount", + "--port", + "--root-path", + "--socket", + "--ssl-ca-certs", + "--ssl-certfile", + "--ssl-keyfile", + "--uds", + "--unix-socket", + "--url", +]); + +function normalizedSecurityText(value: string): string { + return value.replace(/([a-z0-9])([A-Z])/gu, "$1_$2"); +} + +function validatePublicManifestValue(value: unknown): void { + if (typeof value === "string") { + const normalized = normalizedSecurityText(value); + if (SECRET_LIKE_COMPONENT.test(normalized) || SECRET_LIKE_VALUE.test(value)) { + throw new Error("public manifest contains a secret-like field name or value"); + } + if (URL_LIKE_VALUE.test(value)) { + throw new Error("public manifest contains a URL-looking value"); + } + if (value.trim() === "/" || ABSOLUTE_HOST_PATH.test(value)) { + throw new Error("public manifest contains an absolute host path"); + } + return; + } + if (Array.isArray(value)) { + for (const item of value) validatePublicManifestValue(item); + return; + } + if (!value || typeof value !== "object") return; + for (const [name, item] of Object.entries(value)) { + if (SECRET_LIKE_COMPONENT.test(normalizedSecurityText(name))) { + throw new Error("public manifest contains a secret-like field name or value"); + } + validatePublicManifestValue(item); + } +} + +/** Fail closed on obvious private material before a manifest can enter public artifacts. */ +export function validatePublicManifestFields(manifest: ToolDisclosureManifest): void { + for (const flag of manifest.inference.public_vllm_flags) { + for (const token of flag.trim().split(/\s+/u)) { + const name = token.match(/^--[a-z0-9][a-z0-9-]*/iu)?.[0].toLowerCase(); + if (name && PRIVATE_ROUTING_FLAGS.has(name)) { + throw new Error("public_vllm_flags contains a private routing flag"); + } + } + } + validatePublicManifestValue(manifest); +} + +export function validateFrozenManifest(manifest: ToolDisclosureManifest): void { + validatePublicManifestFields(manifest); + const requiredEnvironmentText = [ + manifest.environment.operating_system, + manifest.environment.architecture, + manifest.environment.cpu_model, + manifest.environment.accelerator_type, + manifest.environment.accelerator_model, + manifest.environment.accelerator_architecture, + manifest.environment.accelerator_driver_version, + manifest.environment.accelerator_runtime, + manifest.environment.power_state, + manifest.environment.openshell_version, + ...TOOL_DISCLOSURE_AGENTS.map((agent) => manifest.environment.agent_versions[agent]), + ]; + const requiredInferenceText = [ + manifest.inference.model_id, + manifest.inference.model_revision, + manifest.inference.container_image, + manifest.inference.vllm_version, + manifest.inference.tool_call_parser, + manifest.inference.reasoning_parser, + ]; + if ( + manifest.schema_version !== "nemoclaw.tool_disclosure_performance_test.v1" || + !/^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/u.test(manifest.performance_test_id) || + !Number.isFinite(Date.parse(manifest.created_at)) || + !/^[a-f0-9]{40}$/u.test(manifest.sut.git_sha) || + !/^[a-f0-9]{40}$/u.test(manifest.harness.git_sha) || + !manifest.sut.git_ref.trim() || + !manifest.harness.git_ref.trim() || + requiredEnvironmentText.some((value) => typeof value !== "string" || !value.trim()) || + requiredInferenceText.some((value) => typeof value !== "string" || !value.trim()) || + !Number.isSafeInteger(manifest.environment.cpu_count) || + manifest.environment.cpu_count <= 0 || + !Number.isFinite(manifest.environment.ram_gib) || + manifest.environment.ram_gib <= 0 || + !Number.isSafeInteger(manifest.environment.accelerator_count) || + manifest.environment.accelerator_count < 0 || + !/^sha256:[a-f0-9]{64}$/u.test(manifest.inference.container_digest) || + manifest.inference.api !== "chat-completions" || + manifest.inference.public_vllm_flags.some( + (flag) => typeof flag !== "string" || !flag.trim() || /[\r\n]/u.test(flag), + ) || + new Set(manifest.inference.public_vllm_flags).size !== + manifest.inference.public_vllm_flags.length || + JSON.stringify(manifest).includes("RECORD_BEFORE_EXECUTION") + ) { + throw new Error("manifest is missing immutable claim-grade environment metadata"); + } + const acceleratorMetadata = [ + manifest.environment.accelerator_model, + manifest.environment.accelerator_architecture, + manifest.environment.accelerator_driver_version, + manifest.environment.accelerator_runtime, + ]; + if ( + (manifest.environment.accelerator_count === 0 && + (manifest.environment.accelerator_type !== NO_ACCELERATOR_TYPE || + acceleratorMetadata.some((value) => value !== NOT_APPLICABLE))) || + (manifest.environment.accelerator_count > 0 && + (manifest.environment.accelerator_type === NO_ACCELERATOR_TYPE || + manifest.environment.accelerator_type === NOT_APPLICABLE || + acceleratorMetadata.some((value) => value === NOT_APPLICABLE))) + ) { + throw new Error("manifest accelerator metadata is inconsistent with accelerator count"); + } + if ( + !sameJson(manifest.protocol.agents, TOOL_DISCLOSURE_AGENTS) || + !sameJson(manifest.protocol.modes, TOOL_DISCLOSURE_MODES) || + !sameJson(manifest.protocol.catalog_sizes, STATIC_CATALOG_SIZES) || + manifest.protocol.primary_catalog_size !== 512 || + !sameJson(manifest.protocol.repetitions, { + "small-control": 1, + primary: 5, + "large-stress": 1, + }) || + manifest.protocol.bootstrap_samples !== DEFAULT_BOOTSTRAP_SAMPLES || + manifest.protocol.bootstrap_seed !== DEFAULT_BOOTSTRAP_SEED || + manifest.protocol.noninferiority_margin_percentage_points !== + DEFAULT_NONINFERIORITY_MARGIN_PP || + manifest.protocol.retry_setup_failures !== 1 + ) { + throw new Error("manifest does not match the frozen performance test protocol"); + } + if ( + manifest.inference.temperature !== 0 || + manifest.inference.concurrency !== 1 || + manifest.inference.prefix_caching_enabled || + !manifest.sut.worktree_clean || + !manifest.harness.worktree_clean + ) { + throw new Error("manifest does not record clean frozen execution controls"); + } + if ( + !sameJson( + manifest.campaigns.map((campaign) => campaign.campaign_id), + ["campaign-1", "campaign-2"], + ) || + !sameJson( + manifest.campaigns.map((campaign) => campaign.ordinal), + [1, 2], + ) || + manifest.campaigns.some( + (campaign) => !campaign.fresh_inference_process || !campaign.fresh_sandboxes, + ) + ) { + throw new Error("manifest must record two ordered fresh campaigns"); + } + const imageKeys = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => `${agent}:${mode}:${size}`), + ), + ).sort(); + const images = manifest.environment.sandbox_image_digests ?? {}; + if ( + !sameJson(Object.keys(images).sort(), imageKeys) || + Object.values(images).some((digest) => !/^sha256:[a-f0-9]{64}$/u.test(digest)) + ) { + throw new Error("manifest must record an immutable image digest for every sandbox cell"); + } +} + +function expectedCampaignId(run: ScheduledToolDisclosureRun): string { + return `campaign-${run.campaign}`; +} + +function assertRunMatchesSchedule( + run: ToolDisclosureRun, + expected: ScheduledToolDisclosureRun, + taskKinds: ReadonlyMap, +): void { + const expectedTaskId = expected.task_id ?? "static-capture"; + const fields = [ + ["campaign_id", run.campaign_id, expectedCampaignId(expected)], + ["phase", run.phase, expected.phase], + ["agent", run.agent, expected.agent], + ["mode", run.mode, expected.mode], + ["catalog_size", run.catalog_size, expected.catalog_size], + ["task_id", run.task_id, expectedTaskId], + ["repetition", run.repetition, expected.repetition], + ] as const; + for (const [field, actual, wanted] of fields) { + if (actual !== wanted) throw new Error(`run ${run.run_id} has off-schedule ${field}`); + } + const expectedKind = expected.task_id ? taskKinds.get(expected.task_id) : undefined; + if (run.task_kind !== expectedKind) { + throw new Error(`run ${run.run_id} has an off-schedule task kind`); + } +} + +function assertTerminalMeasurements(run: ToolDisclosureRun): void { + const measurements = run.measurements; + const required = [ + ["initial schema bytes", measurements.initial_tool_schema.serialized_bytes], + ["initial schema tokens", measurements.initial_tool_schema.tokenizer_tokens], + ["prompt tokens", measurements.total_prompt_tokens], + ["completion tokens", measurements.completion_tokens], + ["inference time", measurements.inference_time_ms], + ["end-to-end time", measurements.end_to_end_time_ms], + ] as const; + for (const [label, value] of required) { + if (value === undefined || !Number.isFinite(value) || value < 0) { + throw new Error(`run ${run.run_id} has invalid or missing ${label}`); + } + } + if ( + measurements.initial_tool_schema.serialized_bytes === 0 || + measurements.initial_tool_schema.tokenizer_tokens === 0 || + !Number.isSafeInteger(measurements.model_calls) || + measurements.model_calls < 1 || + !Number.isSafeInteger(measurements.discovery_calls) || + measurements.discovery_calls < 0 + ) { + throw new Error(`run ${run.run_id} has invalid required count measurements`); + } +} + +/** Validate the complete frozen run matrix before any public summary is emitted. */ +export function validateCompleteEvidence(options: { + manifest: ToolDisclosureManifest; + runs: readonly ToolDisclosureRun[]; + schedule: readonly ScheduledToolDisclosureRun[]; + primaryTasks: SyntheticTaskSet; + stressTasks: SyntheticTaskSet; + rawEvidence: readonly SanitizedRunEvidence[]; + catalog: SyntheticCatalog; +}): void { + const { manifest, runs, schedule, primaryTasks, stressTasks, rawEvidence, catalog } = options; + validateFrozenManifest(manifest); + if (!sameJson(manifest.protocol.agents, TOOL_DISCLOSURE_AGENTS)) { + throw new Error("manifest agents do not match the frozen protocol"); + } + if (!sameJson(manifest.protocol.modes, TOOL_DISCLOSURE_MODES)) { + throw new Error("manifest modes do not match the frozen protocol"); + } + if (!sameJson(manifest.protocol.catalog_sizes, STATIC_CATALOG_SIZES)) { + throw new Error("manifest catalog sizes do not match the frozen protocol"); + } + if ( + manifest.protocol.primary_catalog_size !== 512 || + !sameJson(manifest.protocol.repetitions, { + "small-control": 1, + primary: 5, + "large-stress": 1, + }) + ) { + throw new Error("manifest repetitions do not match the frozen protocol"); + } + if ( + manifest.protocol.bootstrap_samples !== DEFAULT_BOOTSTRAP_SAMPLES || + manifest.protocol.bootstrap_seed !== DEFAULT_BOOTSTRAP_SEED || + manifest.protocol.noninferiority_margin_percentage_points !== + DEFAULT_NONINFERIORITY_MARGIN_PP || + manifest.protocol.retry_setup_failures !== 1 + ) { + throw new Error("manifest statistics or retry settings do not match the frozen protocol"); + } + if ( + manifest.inference.temperature !== 0 || + manifest.inference.concurrency !== 1 || + manifest.inference.prefix_caching_enabled + ) { + throw new Error("manifest inference controls do not match the frozen protocol"); + } + if (!manifest.sut.worktree_clean || !manifest.harness.worktree_clean) { + throw new Error("public performance test evidence requires a clean SUT and harness worktree"); + } + const imageKeys = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => `${agent}:${mode}:${size}`), + ), + ).sort(); + const recordedImageKeys = Object.keys(manifest.environment.sandbox_image_digests ?? {}).sort(); + if ( + !sameJson(recordedImageKeys, imageKeys) || + recordedImageKeys.some( + (key) => + !/^sha256:[a-f0-9]{64}$/u.test(manifest.environment.sandbox_image_digests?.[key] ?? ""), + ) + ) { + throw new Error("manifest must record an immutable image digest for every sandbox cell"); + } + if ( + !sameJson( + manifest.campaigns.map((campaign) => campaign.campaign_id), + ["campaign-1", "campaign-2"], + ) || + manifest.campaigns.some( + (campaign) => !campaign.fresh_inference_process || !campaign.fresh_sandboxes, + ) + ) { + throw new Error("manifest must record two fresh frozen campaigns"); + } + + const expectedSchedule = buildToolDisclosureSchedule({ + primaryTaskIds: primaryTasks.tasks.map((task) => task.id), + stressTaskIds: stressTasks.tasks.map((task) => task.id), + seed: manifest.protocol.execution_seed, + }); + if (!sameJson(schedule, expectedSchedule)) { + throw new Error("schedule.json does not match the deterministic frozen schedule"); + } + const allTasks = [...primaryTasks.tasks, ...stressTasks.tasks]; + const expectedProtocolTasks = allTasks.map((task) => ({ + task_id: task.id, + kind: task.kind, + })); + if (!sameJson(manifest.protocol.tasks, expectedProtocolTasks)) { + throw new Error("manifest tasks do not match the frozen task artifacts"); + } + const taskKinds = new Map(allTasks.map((task) => [task.id, task.kind] as const)); + const tasksById = new Map(allTasks.map((task) => [task.id, task] as const)); + const expectedById = new Map(schedule.map((run) => [run.run_id, run] as const)); + if (expectedById.size !== schedule.length) throw new Error("schedule contains duplicate run IDs"); + const observedById = new Map(); + if (rawEvidence.length !== runs.length) { + throw new Error("raw-events.jsonl and runs.jsonl have different record counts"); + } + let scheduleCursor = 0; + for (const [index, run] of runs.entries()) { + if (schedule[scheduleCursor]?.run_id !== run.run_id) { + throw new Error(`runs.jsonl is out of frozen schedule order at ${run.run_id}`); + } + const expected = expectedById.get(run.run_id); + if (!expected) throw new Error(`runs.jsonl contains unknown run ID ${run.run_id}`); + assertRunMatchesSchedule(run, expected, taskKinds); + const raw = rawEvidence[index]; + if (raw.run_id !== run.run_id) throw new Error(`raw evidence order differs at ${run.run_id}`); + const task = expected.task_id ? tasksById.get(expected.task_id) : undefined; + const rebuilt = assembleToolDisclosureRun({ + manifest, + scheduled: expected, + task, + calls: raw.calls, + recorderEvents: raw.recorder_events, + invocation: { + ...raw.invocation, + final_output: + raw.final_oracles_present && task ? task.expected_final_includes.join(" ") : "", + }, + initialSchemaTokens: raw.initial_schema_tokens, + ...(raw.prompt_tokens === undefined ? {} : { promptTokens: raw.prompt_tokens }), + ...(raw.completion_tokens === undefined ? {} : { completionTokens: raw.completion_tokens }), + ...(raw.failure_outcome ? { failureOutcome: raw.failure_outcome } : {}), + }); + if (!sameJson(rebuilt, run)) throw new Error(`run ${run.run_id} does not match raw evidence`); + if (run.outcome !== "setup-error") { + const visible = raw.recorder_events.find( + (event) => event.model_call_sequence === 1, + )?.tool_names; + if (!visible) throw new Error(`run ${run.run_id} has no initial visible-tool evidence`); + const synthetic = catalog.tools + .slice(0, run.catalog_size) + .map((tool) => tool.definition.function.name); + const exposed = (name: string): boolean => + visible.some( + (candidate) => + candidate === name || candidate.endsWith(`__${name}`) || candidate.endsWith(`_${name}`), + ); + if (run.mode === "direct" && synthetic.some((name) => !exposed(name))) { + throw new Error(`direct run ${run.run_id} did not expose the complete synthetic catalog`); + } + if (run.mode === "progressive" && synthetic.some(exposed)) { + throw new Error( + `progressive run ${run.run_id} exposed a deferred synthetic tool initially`, + ); + } + } + if (run.execution_seed !== manifest.protocol.execution_seed) { + throw new Error(`run ${run.run_id} has the wrong execution seed`); + } + const observed = observedById.get(run.run_id) ?? []; + if (observed.some((candidate) => sameJson(candidate, run))) { + throw new Error(`runs.jsonl contains an exact duplicate for ${run.run_id}`); + } + observed.push(run); + observedById.set(run.run_id, observed); + if (run.outcome !== "setup-error") scheduleCursor += 1; + } + if (scheduleCursor !== schedule.length) throw new Error("runs.jsonl ended before the schedule"); + + for (const expected of schedule) { + const observed = observedById.get(expected.run_id) ?? []; + const setupFailures = observed.filter((run) => run.outcome === "setup-error"); + const terminal = observed.filter((run) => run.outcome !== "setup-error"); + if (terminal.length !== 1) { + throw new Error(`run ${expected.run_id} must have exactly one terminal record`); + } + if (setupFailures.length > manifest.protocol.retry_setup_failures) { + throw new Error(`run ${expected.run_id} exceeds the setup-failure retry allowance`); + } + assertTerminalMeasurements(terminal[0]); + } +} diff --git a/test/brev-nightly-workflow.test.ts b/test/brev-nightly-workflow.test.ts index 7cc0693178..2f8aef763f 100644 --- a/test/brev-nightly-workflow.test.ts +++ b/test/brev-nightly-workflow.test.ts @@ -96,6 +96,7 @@ describe("Brev nightly workflow contract", () => { "pull-requests": "read", }); expect(checkout?.with?.["persist-credentials"]).toBe(false); + expect(checkout?.with?.ref).toBe("${{ env.RESOLVED_BRANCH || inputs.branch || github.ref }}"); expect(resolveBranch?.env?.PR_NUMBER).toBe("${{ inputs.pr_number }}"); expect(resolveBranch?.run).not.toContain("gh pr view ${{"); expect(validation?.outputs?.tested_sha).toBe("${{ steps.tested-ref.outputs.sha }}"); @@ -129,7 +130,7 @@ describe("Brev nightly workflow contract", () => { expect(validation?.env?.TEST_SUITE).toBe("${{ inputs.test_suite }}"); expect(validation?.run).toContain( - "full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all", + "full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|tool-disclosure-performance-smoke|gpu|all", ); expect(validation?.run).toContain("exit 1"); expect(steps.indexOf(validation as NonNullable)).toBeLessThan( @@ -137,6 +138,54 @@ describe("Brev nightly workflow contract", () => { ); }); + it("collects and scans Brev performance smoke evidence before deleting the VM", () => { + const steps = branchValidation.jobs?.["e2e-branch-validation"]?.steps ?? []; + const prerequisite = steps.find( + (step) => step.name === "Install and verify Brev performance smoke tunnel prerequisite", + ); + const collect = steps.find( + (step) => step.name === "Collect Brev tool-disclosure performance smoke artifacts", + ); + const scan = steps.find( + (step) => step.name === "Scan Brev tool-disclosure performance smoke artifacts", + ); + const upload = steps.find( + (step) => step.name === "Upload Brev tool-disclosure performance smoke artifacts", + ); + const cleanup = steps.find((step) => step.name === "Delete Brev instance"); + const run = steps.find((step) => step.name === "Run ephemeral Brev E2E"); + + expect(prerequisite?.if).toContain("tool-disclosure-performance-smoke"); + expect(prerequisite?.env).toEqual({ + CLOUDFLARED_VERSION: "2026.6.1", + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526", + }); + expect(prerequisite?.run).toContain("sha256sum -c -"); + expect(prerequisite?.run).toContain("dpkg-deb -f"); + expect(run?.env).toMatchObject({ + BREV_MIN_VCPU: "${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '8' || '' }}", + BREV_MIN_RAM: "${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '32' || '' }}", + BREV_MIN_DISK: + "${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '100' || '' }}", + }); + expect(collect?.if).toContain("tool-disclosure-performance-smoke"); + expect(collect?.env?.TEST_OUTCOME).toBe("${{ steps.brev-test.outcome }}"); + expect(collect?.run).toContain("/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"); + expect(collect?.run).toContain("tar -C"); + expect(collect?.run).toContain('if [ "$TEST_OUTCOME" = "success" ]'); + expect(scan?.env?.NVIDIA_INFERENCE_API_KEY).toBe("${{ secrets.NVIDIA_INFERENCE_API_KEY }}"); + expect(scan?.run).toContain("grep -R -I -l -F"); + expect(upload?.uses).toBe("actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"); + expect(upload?.with).toMatchObject({ + name: "e2e-brev-tool-disclosure-performance-smoke-${{ github.run_attempt }}", + "if-no-files-found": "ignore", + "retention-days": 14, + }); + expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( + steps.indexOf(upload as NonNullable), + ); + }); + it("runs stateful messaging targets on separate fresh instances", () => { expect(nightly.jobs?.["brev-nightly-e2e"]?.strategy?.matrix?.test_suite).toEqual([ "all", diff --git a/test/brev-remote-vitest.test.ts b/test/brev-remote-vitest.test.ts index 988429d8a6..8ca10acfff 100644 --- a/test/brev-remote-vitest.test.ts +++ b/test/brev-remote-vitest.test.ts @@ -13,11 +13,16 @@ import { BREV_MESSAGING_PROVIDER_TIMEOUT_MS, BREV_REMOTE_WRAPPER_GRACE_MS, BREV_SECURITY_SUITE_TIMEOUT_MS, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS, BREV_WORKFLOW_OWNERSHIP_ENV, brevSuiteHarnessSandboxName, brevSuiteNeedsHarnessSandbox, brevWorkflowOwnsInstance, + buildBrevMcpSshForwardArgs, buildBrevRemoteVitestCommand, + buildBrevSshForwardEnvironment, } from "../tools/e2e/brev-remote-vitest.mts"; const TARGET = "test/e2e/live/credential-sanitization.test.ts"; @@ -98,6 +103,8 @@ describe("Brev remote Vitest command", () => { expect(BREV_SECURITY_SUITE_TIMEOUT_MS).toBe(20 * 60_000); expect(BREV_MESSAGING_PROVIDER_TIMEOUT_MS).toBe(70 * 60_000); expect(BREV_MESSAGING_COMPAT_TIMEOUT_MS).toBe(40 * 60_000); + expect(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS).toBe(55 * 60_000); + expect(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT).toBeGreaterThanOrEqual(1_024); expect(BREV_REMOTE_WRAPPER_GRACE_MS).toBe(120_000); }); @@ -108,15 +115,59 @@ describe("Brev remote Vitest command", () => { expect(brevWorkflowOwnsInstance({})).toBe(false); }); + it("keeps inference and workflow credentials out of the MCP SSH forward", () => { + expect( + buildBrevSshForwardEnvironment({ + PATH: "/usr/bin", + HOME: "/home/runner", + LC_ALL: "C.UTF-8", + SSH_AUTH_SOCK: "/tmp/agent.sock", + NVIDIA_INFERENCE_API_KEY: "secret-inference-key", + GITHUB_TOKEN: "secret-workflow-token", + }), + ).toEqual({ + PATH: "/usr/bin", + HOME: "/home/runner", + LC_ALL: "C.UTF-8", + }); + expect(buildBrevMcpSshForwardArgs("e2e-performance-smoke", 45_123)).toEqual([ + "-N", + "-T", + "-o", + "BatchMode=yes", + "-o", + "ForwardAgent=no", + "-o", + "StrictHostKeyChecking=no", + "-o", + "LogLevel=ERROR", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "ConnectTimeout=15", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=4", + "-L", + "127.0.0.1:45123:127.0.0.1:43117", + "e2e-performance-smoke", + ]); + }); + it("does not seed shared harness state for suites that own their sandbox lifecycle", () => { expect(brevSuiteNeedsHarnessSandbox("all")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("full")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("gpu")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("messaging-compatible-endpoint")).toBe(false); expect(brevSuiteNeedsHarnessSandbox("messaging-providers")).toBe(false); + expect(brevSuiteNeedsHarnessSandbox(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE)).toBe(false); expect(brevSuiteHarnessSandboxName("all")).toBeUndefined(); expect(brevSuiteHarnessSandboxName("messaging-compatible-endpoint")).toBeUndefined(); expect(brevSuiteHarnessSandboxName("messaging-providers")).toBeUndefined(); + expect( + brevSuiteHarnessSandboxName(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE), + ).toBeUndefined(); }); it("preserves harness onboarding for single-target suites", () => { diff --git a/test/cloudflared-update-check-workflow.test.ts b/test/cloudflared-update-check-workflow.test.ts index 99ee32e346..65635b76c9 100644 --- a/test/cloudflared-update-check-workflow.test.ts +++ b/test/cloudflared-update-check-workflow.test.ts @@ -40,7 +40,7 @@ function pinValues(source: string, name: string): string[] { function writePinFixture(file: string, version: string, sha256: string): void { fs.writeFileSync( file, - ["one", "two", "three"] + ["one", "two", "three", "four"] .map( (job) => ` ${job}:\n env:\n CLOUDFLARED_VERSION: "${version}"\n CLOUDFLARED_DEB_SHA256: "${sha256}"`, @@ -140,11 +140,11 @@ describe("cloudflared update-check workflow contract", () => { expect(check?.run).toBe("bash scripts/checks/check-cloudflared-update.sh"); }); - it("extracts exactly three identical reviewed version and SHA256 pins", () => { + it("extracts exactly four identical reviewed version and SHA256 pins", () => { const versions = pinValues(e2e, "CLOUDFLARED_VERSION"); const hashes = pinValues(e2e, "CLOUDFLARED_DEB_SHA256"); - expect(versions).toHaveLength(3); - expect(hashes).toHaveLength(3); + expect(versions).toHaveLength(4); + expect(hashes).toHaveLength(4); expect(new Set(versions).size).toBe(1); expect(new Set(hashes).size).toBe(1); expect(versions[0]).toMatch(/^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$/u); @@ -188,7 +188,7 @@ describe("cloudflared update-check workflow contract", () => { ); expect(fixture.result.stderr).toContain("CLOUDFLARED_VERSION lines:"); expect(fixture.result.stderr).toContain("CLOUDFLARED_DEB_SHA256 lines:"); - expect(fixture.result.stderr).toContain("Set all three version/SHA256 pairs"); + expect(fixture.result.stderr).toContain("Set all four version/SHA256 pairs"); } finally { fs.rmSync(fixture.tempDir, { recursive: true, force: true }); } diff --git a/test/dcode-start-keepalive.test.ts b/test/dcode-start-keepalive.test.ts index 00412f9c71..9e68e3bdf8 100644 --- a/test/dcode-start-keepalive.test.ts +++ b/test/dcode-start-keepalive.test.ts @@ -24,6 +24,7 @@ function makeStartFixture(): string { const scriptPath = path.join(tempDir, "start.sh"); const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); + const toolDisclosureFile = path.join(tempDir, "trusted-tool-disclosure"); const fixture = fs .readFileSync(START_SCRIPT, "utf8") .replace( @@ -34,14 +35,20 @@ function makeStartFixture(): string { 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', `readonly MANAGED_PROXY_PORT_FILE="${portFile}"`, ) + .replace( + 'readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure"', + `readonly MANAGED_TOOL_DISCLOSURE_FILE="${toolDisclosureFile}"`, + ) .replace( "readonly MANAGED_PROXY_OWNER_UID=0", `readonly MANAGED_PROXY_OWNER_UID=${process.getuid?.() ?? 0}`, ); fs.writeFileSync(hostFile, "10.200.0.1\n"); fs.writeFileSync(portFile, "3128\n"); + fs.writeFileSync(toolDisclosureFile, "progressive\n"); fs.chmodSync(hostFile, 0o444); fs.chmodSync(portFile, 0o444); + fs.chmodSync(toolDisclosureFile, 0o444); fs.writeFileSync(scriptPath, fixture); fs.chmodSync(scriptPath, 0o755); tempDirs.push(tempDir); diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index b40f3c7247..4fbaebd698 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -28,7 +28,8 @@ * Optional env vars: * TEST_SUITE — which test to run: full (default), deploy-cli, gpu, * credential-sanitization, telegram-injection, messaging-providers, - * messaging-compatible-endpoint, dashboard-remote-bind, all + * messaging-compatible-endpoint, dashboard-remote-bind, + * tool-disclosure-performance-smoke, all * BREV_MIN_VCPU — Minimum vCPUs for CPU instance (default: 4) * BREV_MIN_RAM — Minimum RAM in GB for CPU instance (default: 16) * BREV_PROVIDER — Cloud provider filter for brev search (default: gcp for CPU, any for GPU) @@ -49,19 +50,41 @@ * TELEGRAM_CHAT_ID_E2E — Telegram chat ID for optional sendMessage test */ -import { execFileSync, execSync, type StdioOptions, spawnSync } from "node:child_process"; +import { + type ChildProcess, + execFileSync, + execSync, + type StdioOptions, + spawn, + spawnSync, +} from "node:child_process"; +import net from "node:net"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + type QuickTunnel, + startQuickTunnel, +} from "../../scripts/performance/tool-disclosure/quick-tunnel"; +import { + PERFORMANCE_SMOKE_MCP_PORT_ENV, + PERFORMANCE_SMOKE_MCP_URL_ENV, +} from "../../scripts/performance/tool-disclosure/smoke-mcp-transport"; import { shellQuote } from "../../src/lib/core/shell-quote"; import { BREV_MESSAGING_COMPAT_TIMEOUT_MS, BREV_MESSAGING_PROVIDER_TIMEOUT_MS, BREV_REMOTE_WRAPPER_GRACE_MS, BREV_SECURITY_SUITE_TIMEOUT_MS, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS, brevSuiteHarnessSandboxName, brevSuiteNeedsHarnessSandbox, brevWorkflowOwnsInstance, + buildBrevMcpSshForwardArgs, buildBrevRemoteVitestCommand, + buildBrevSshForwardEnvironment, } from "../../tools/e2e/brev-remote-vitest.mts"; // Instance configuration @@ -76,6 +99,8 @@ const TEST_SUITE = process.env.TEST_SUITE || "full"; const REPO_DIR = path.resolve(import.meta.dirname, "../.."); const CLI_PATH = path.join(REPO_DIR, "bin", "nemoclaw.js"); const GPU_TEST_SUITE = TEST_SUITE === "gpu"; +const TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE = + TEST_SUITE === BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE; const BREV_PROVIDER = process.env.BREV_PROVIDER || (GPU_TEST_SUITE ? "" : "gcp"); const BREV_CREATE_TIMEOUT_SECONDS = parseInt( process.env.BREV_CREATE_TIMEOUT_SECONDS || (GPU_TEST_SUITE ? "1200" : "180"), @@ -117,9 +142,15 @@ const SETUP_SCRIPT_PATH = path.join(REPO_DIR, "scripts", "brev-launchable-ci-cpu // Sentinel file written by brev-launchable-ci-cpu.sh when setup is complete. // More reliable than grepping log files. const LAUNCHABLE_SENTINEL = "/var/run/nemoclaw-launchable-ready"; +const PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE = + "/tmp/nemoclaw-tool-disclosure-performance-mcp-placeholder.pid"; +const PERFORMANCE_SMOKE_MCP_PLACEHOLDER_LOG_FILE = + "/tmp/nemoclaw-tool-disclosure-performance-mcp-placeholder.log"; let remoteDir = ""; let instanceCreated = false; +let performanceSmokeMcpForward: ChildProcess | undefined; +let performanceSmokeMcpTunnel: QuickTunnel | undefined; const STREAM_STDIO: StdioOptions = ["inherit", "inherit", "inherit"]; const CAPTURE_STDIO: StdioOptions = ["pipe", "pipe", "pipe"]; @@ -288,6 +319,24 @@ function sshEnv( // sandbox creation while auto-loading a very large default Ollama model. envParts.push(`export NEMOCLAW_MODEL='${shellEscape(gpuE2eModel)}'`); } + if (TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE) { + if (!performanceSmokeMcpTunnel) { + throw new Error("Brev performance smoke MCP tunnel is not ready"); + } + const testedSha = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: REPO_DIR, + encoding: "utf8", + }).trim(); + if (!/^[a-f0-9]{40}$/u.test(testedSha)) { + throw new Error("Brev performance smoke requires an exact local git SHA"); + } + envParts.push( + `export E2E_ARTIFACT_DIR='${shellEscape(BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR)}'`, + `export GITHUB_SHA='${testedSha}'`, + `export ${PERFORMANCE_SMOKE_MCP_PORT_ENV}='${BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT}'`, + `export ${PERFORMANCE_SMOKE_MCP_URL_ENV}='${shellEscape(performanceSmokeMcpTunnel.mcpUrl)}'`, + ); + } // Forward optional messaging tokens for the messaging-providers test for (const key of [ "TELEGRAM_BOT_TOKEN", @@ -455,6 +504,16 @@ function printRemoteFailureDiagnostics(): void { `PATH=$HOME/.local/bin:$PATH openshell sandbox list 2>&1 || true`, `echo "--- docker ps ---"`, `sg docker -c 'docker ps -a --filter label=openshell.ai/managed-by=openshell' 2>&1 || true`, + `echo "--- memory ---"`, + `free -h 2>&1 || true`, + `echo "--- disk ---"`, + `df -h 2>&1 || true`, + `echo "--- docker service ---"`, + `sudo systemctl status docker --no-pager 2>&1 || true`, + `echo "--- recent docker journal ---"`, + `sudo journalctl -u docker --since "15 minutes ago" --no-pager 2>&1 | tail -200 || true`, + `echo "--- recent kernel messages ---"`, + `sudo dmesg 2>&1 | tail -100 || true`, `echo "--- openshell gateway log ---"`, `tail -200 "$HOME/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.log" 2>&1 || true`, `latest="$(find "$HOME/.nemoclaw/onboard-failures" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -1)"`, @@ -766,6 +825,133 @@ function prepareGpuDockerRuntime(elapsed: () => string): void { console.log(`[${elapsed()}] NVIDIA Docker runtime ready`); } +function performanceSmokePlaceholderSource(): string { + return [ + "from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer", + "class Handler(BaseHTTPRequestHandler):", + " def do_HEAD(self):", + " self.send_response(405)", + " self.end_headers()", + " def do_GET(self):", + " self.send_response(405)", + " self.end_headers()", + " def log_message(self, format, *args):", + " pass", + `ThreadingHTTPServer((\"127.0.0.1\", ${BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT}), Handler).serve_forever()`, + ].join("\n"); +} + +function startPerformanceSmokeMcpPlaceholder(): void { + ssh( + [ + "set -euo pipefail", + `rm -f ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE} ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_LOG_FILE}`, + `nohup python3 -c ${shellQuote(performanceSmokePlaceholderSource())} ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_LOG_FILE} 2>&1 &`, + "placeholder_pid=$!", + `printf '%s\\n' \"$placeholder_pid\" >${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE}`, + "sleep 1", + 'kill -0 "$placeholder_pid"', + ].join("\n"), + { timeout: 15_000 }, + ); +} + +function stopPerformanceSmokeMcpPlaceholder(): void { + ssh( + [ + "set +e", + `if test -s ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE}; then`, + `placeholder_pid=\"$(cat ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE})\"`, + 'kill "$placeholder_pid" 2>/dev/null', + "for attempt in 1 2 3 4 5; do", + 'if ! kill -0 "$placeholder_pid" 2>/dev/null; then break; fi', + "sleep 1", + "done", + 'kill -9 "$placeholder_pid" 2>/dev/null', + "fi", + `rm -f ${PERFORMANCE_SMOKE_MCP_PLACEHOLDER_PID_FILE}`, + ].join("\n"), + { timeout: 15_000 }, + ); +} + +function reserveLoopbackPort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + reject(new Error("failed to reserve a loopback port for the Brev MCP forward")); + return; + } + const port = address.port; + server.close((error) => (error ? reject(error) : resolve(port))); + }); + }); +} + +function canConnectLoopback(port: number): Promise { + return new Promise((resolve) => { + const socket = net.createConnection({ host: "127.0.0.1", port }); + const finish = (connected: boolean): void => { + socket.removeAllListeners(); + socket.destroy(); + resolve(connected); + }; + socket.setTimeout(500); + socket.once("connect", () => finish(true)); + socket.once("error", () => finish(false)); + socket.once("timeout", () => finish(false)); + }); +} + +async function stopManagedChild(child: ChildProcess | undefined): Promise { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const closed = new Promise((resolve) => child.once("close", () => resolve())); + child.kill("SIGTERM"); + await Promise.race([closed, new Promise((resolve) => setTimeout(resolve, 5_000))]); + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); +} + +async function startPerformanceSmokeSshForward(localPort: number): Promise { + const child = spawn("ssh", buildBrevMcpSshForwardArgs(requireInstanceName(), localPort), { + env: buildBrevSshForwardEnvironment(process.env), + stdio: ["ignore", "ignore", "inherit"], + }); + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error("Brev MCP SSH forward exited before readiness"); + } + if (await canConnectLoopback(localPort)) return child; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + await stopManagedChild(child); + throw new Error("Brev MCP SSH forward did not become ready"); +} + +async function prepareToolDisclosurePerformanceSmoke(elapsed: () => string): Promise { + console.log(`[${elapsed()}] Preparing runner-relayed MCP tunnel...`); + startPerformanceSmokeMcpPlaceholder(); + const localPort = await reserveLoopbackPort(); + try { + performanceSmokeMcpForward = await startPerformanceSmokeSshForward(localPort); + performanceSmokeMcpTunnel = await startQuickTunnel({ + port: localPort, + protocol: "auto", + timeoutMs: 90_000, + }); + } catch (error) { + await stopManagedChild(performanceSmokeMcpForward); + performanceSmokeMcpForward = undefined; + stopPerformanceSmokeMcpPlaceholder(); + throw error; + } + console.log(`[${elapsed()}] Runner-relayed MCP tunnel ready`); +} + /** * Bootstrap the launchable environment on the remote VM: * rsync branch code, install deps, build plugin, and npm link the CLI. @@ -1094,7 +1280,7 @@ describe("Brev GPU runtime setup", () => { }); describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { - beforeAll(() => { + beforeAll(async () => { const bootstrapStart = Date.now(); const elapsed = () => `${Math.round((Date.now() - bootstrapStart) / 1000)}s`; @@ -1125,6 +1311,10 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { const result = bootstrapLaunchable(elapsed); remoteDir = result.remoteDir; + if (TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE) { + await prepareToolDisclosurePerformanceSmoke(elapsed); + } + if (result.needsOnboard) { pollForSandboxReady(elapsed); writeManualRegistry(elapsed); @@ -1149,7 +1339,16 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { console.log(`[${elapsed()}] beforeAll complete — total bootstrap time: ${elapsed()}`); }, 2_700_000); // 45 min - afterAll(() => { + afterAll(async () => { + if (performanceSmokeMcpTunnel) { + await performanceSmokeMcpTunnel.close(); + performanceSmokeMcpTunnel = undefined; + } + await stopManagedChild(performanceSmokeMcpForward); + performanceSmokeMcpForward = undefined; + if (instanceCreated && TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE) { + stopPerformanceSmokeMcpPlaceholder(); + } if (!instanceCreated) return; const keepAlive = process.env.KEEP_ALIVE === "true"; const workflowOwnsInstance = brevWorkflowOwnsInstance(); @@ -1188,6 +1387,20 @@ describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { 1_800_000, ); + it.runIf(TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE)( + "tool-disclosure performance smoke passes on Brev CPU VM", + () => { + stopPerformanceSmokeMcpPlaceholder(); + const output = runRemoteVitest( + "e2e-live", + "test/e2e/live/tool-disclosure-performance-smoke.test.ts", + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS, + ); + expectVitestPassed(output); + }, + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS + BREV_REMOTE_WRAPPER_GRACE_MS, + ); + it.runIf(TEST_SUITE === "credential-sanitization" || TEST_SUITE === "all")( "credential sanitization suite passes on remote VM", () => { diff --git a/test/e2e/live/tool-disclosure-performance-smoke.test.ts b/test/e2e/live/tool-disclosure-performance-smoke.test.ts new file mode 100644 index 0000000000..dcc434b4a7 --- /dev/null +++ b/test/e2e/live/tool-disclosure-performance-smoke.test.ts @@ -0,0 +1,629 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import os from "node:os"; +import { + DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, + generateCatalogPrefix, + generateSyntheticCatalog, +} from "../../../scripts/performance/tool-disclosure/catalog"; +import { + createOpenAIChatTaskDecomposer, + PortableHashingTextEmbedder, +} from "../../../scripts/performance/tool-disclosure/compositional-tool-routing-adapters"; +import { runCompositionalRoutingAcceptance } from "../../../scripts/performance/tool-disclosure/compositional-tool-routing-run"; +import { CompositionalToolRoutingTransform } from "../../../scripts/performance/tool-disclosure/compositional-tool-routing-transform"; +import { + buildAgentDriverCommand, + extractFinalAssistantOutput, +} from "../../../scripts/performance/tool-disclosure/drivers"; +import { gradeTaskRun } from "../../../scripts/performance/tool-disclosure/grading"; +import { SyntheticMcpServer } from "../../../scripts/performance/tool-disclosure/mcp-server"; +import { startQuickTunnel } from "../../../scripts/performance/tool-disclosure/quick-tunnel"; +import { createToolDisclosureRecordingProxy } from "../../../scripts/performance/tool-disclosure/recorder"; +import type { ToolDisclosureMode } from "../../../scripts/performance/tool-disclosure/schedule"; +import { + resolvePerformanceSmokeMcpTransport, + waitForPerformanceSmokeMcpEndpoint, +} from "../../../scripts/performance/tool-disclosure/smoke-mcp-transport"; +import { generatePrimaryTaskSet } from "../../../scripts/performance/tool-disclosure/tasks"; +import { LOCAL_INFERENCE_TIMEOUT_SECS } from "../../../src/lib/onboard/env"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; + +const AGENT = "langchain-deepagents-code" as const; +const CATALOG_SIZE = 64; +const TASK_ID = "primary-single-01"; +const MODES = ["progressive", "direct"] as const satisfies readonly ToolDisclosureMode[]; +const MCP_SERVER_NAME = "performance-test"; +const MCP_TOKEN_ENV = "TOOL_DISCLOSURE_PERFORMANCE_TEST_MCP_TOKEN"; +const OPENSHELL_DOCKER_NETWORK = + process.env.OPENSHELL_DOCKER_NETWORK_NAME?.trim() || "openshell-docker"; +const PRIVATE_BRIDGE_PROBE_IMAGE = + "busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662"; +const TEST_TIMEOUT_MS = 50 * 60_000; +const ROUTING_DECOMPOSER_MAX_ATTEMPTS = 2; +const ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS = 120_000; +const ROUTE_ONLY_RUN_TIMEOUT_MS = 900_000; +const ROUTED_PROXY_REQUEST_TIMEOUT_MS = 720_000; +const ROUTED_GATEWAY_REQUEST_TIMEOUT_MS = 840_000; +const ROUTED_AGENT_INVOCATION_TIMEOUT_MS = 900_000; +const RESTORED_GATEWAY_REQUEST_TIMEOUT_MS = LOCAL_INFERENCE_TIMEOUT_SECS * 1_000; +const MANAGED_MCP_SNAPSHOT_PROBE = [ + "import os", + "from deepagents_code import _nemoclaw_managed as managed", + 'payload = b"managed-mcp-snapshot-probe\\n"', + "descriptor, binding = managed._managed_mcp_snapshot(payload)", + 'assert binding["kind"] in {managed._MCP_SEALED_KIND, managed._MCP_ANONYMOUS_KIND}', + "assert managed._read_bound_managed_mcp_descriptor(descriptor, binding) == payload", + "os.close(descriptor)", + 'print("managed-mcp-snapshot-ok:" + binding["kind"])', +].join("; "); + +function sandboxName(mode: ToolDisclosureMode): string { + return `e2e-tool-disclosure-performance-${mode}`; +} + +function hostEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +function gitSha(): string { + const sha = + process.env.GITHUB_SHA?.trim() || + execFileSync("git", ["rev-parse", "HEAD"], { + encoding: "utf8", + }).trim(); + return /^[a-f0-9]{40}$/u.test(sha) + ? sha + : (() => { + throw new Error("tool-disclosure performance smoke requires a git SHA"); + })(); +} + +test("tool disclosure hosted-inference performance smoke completes one frozen task in both modes", { + timeout: TEST_TIMEOUT_MS, +}, async ({ artifacts, cleanup, host, sandbox, secrets }) => { + const hosted = requireHostedInferenceConfig(secrets); + const bearerToken = randomBytes(32).toString("hex"); + const routingIngressToken = randomBytes(32).toString("hex"); + artifacts.addRedactionValues([ + hosted.apiKey, + hosted.endpointUrl, + bearerToken, + routingIngressToken, + ]); + + const routingCredentialEnv = "NEMOCLAW_COMPOSITIONAL_ROUTING_SMOKE_API_KEY"; + const previousRoutingCredential = process.env[routingCredentialEnv]; + process.env[routingCredentialEnv] = hosted.apiKey; + let routingAcceptance: Awaited>; + try { + routingAcceptance = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: hosted.endpointUrl, + model: hosted.model, + revision: process.env.NEMOCLAW_MODEL_REVISION?.trim() || "unreported", + api_key_env: routingCredentialEnv, + allow_remote: true, + reasoning_control: "enable_thinking_false", + json_object_response: true, + max_attempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, + }, + embedding: { kind: "portable" }, + timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, + run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, + }); + } finally { + previousRoutingCredential === undefined + ? delete process.env[routingCredentialEnv] + : (process.env[routingCredentialEnv] = previousRoutingCredential); + } + await artifacts.writeJson("compositional-routing-acceptance.json", routingAcceptance); + expect(routingAcceptance.acceptance_passed).toBe(true); + + const catalog = generateSyntheticCatalog({ + seed: DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, + }); + const catalogPrefix = generateCatalogPrefix(catalog, CATALOG_SIZE); + const primaryTasks = generatePrimaryTaskSet(catalog); + const mcpTransport = resolvePerformanceSmokeMcpTransport(); + const task = primaryTasks.tasks.find((candidate) => candidate.id === TASK_ID); + expect(task, `frozen task ${TASK_ID} is missing`).toBeDefined(); + const frozenTask = task as NonNullable; + expect( + frozenTask.min_catalog_size, + `frozen task ${TASK_ID} does not fit the performance smoke catalog`, + ).toBeLessThanOrEqual(catalogPrefix.size); + + await artifacts.writeJson("scenario.json", { + id: "tool-disclosure-performance-smoke", + boundary: + "tool-disclosure performance smoke with two real Deep Agents Code sandboxes, hosted inference, and synthetic MCP", + claim_eligible: false, + agent: AGENT, + modes: MODES, + catalog_size: CATALOG_SIZE, + task_id: TASK_ID, + model_id: hosted.model, + mcp_transport: mcpTransport.kind, + }); + + const mcp = new SyntheticMcpServer(catalogPrefix, bearerToken); + const mcpAddress = await mcp.start( + mcpTransport.kind === "external-ssh-forward" ? mcpTransport.listenPort : 0, + ); + cleanup.add("stop tool-disclosure performance synthetic MCP server", () => mcp.stop()); + const localTunnel = + mcpTransport.kind === "local-quick-tunnel" + ? await startQuickTunnel({ port: mcpAddress.port }) + : undefined; + if (localTunnel) { + cleanup.add("stop tool-disclosure performance MCP quick tunnel", () => localTunnel.close()); + } + const mcpUrl = + mcpTransport.kind === "external-ssh-forward" ? mcpTransport.mcpUrl : localTunnel?.mcpUrl; + expect(mcpUrl, "performance smoke MCP URL was not established").toBeDefined(); + const configuredMcpUrl = mcpUrl as string; + artifacts.addRedactionValues([configuredMcpUrl]); + await waitForPerformanceSmokeMcpEndpoint(configuredMcpUrl); + + for (const mode of MODES) { + const name = sandboxName(mode); + cleanup.add(`destroy ${mode} tool-disclosure performance smoke sandbox`, () => + host.bestEffortCleanupSandbox(name, { + artifactName: `cleanup-${mode}-tool-disclosure-sandbox`, + env: hostEnv(), + }), + ); + await host.cleanupSandbox(name, { + artifactName: `preclean-${mode}-tool-disclosure-sandbox`, + env: hostEnv(), + }); + + const onboard = await host.nemoclaw( + [ + "onboard", + "--non-interactive", + "--fresh", + "--yes", + "--yes-i-accept-third-party-software", + "--agent", + AGENT, + "--name", + name, + "--tool-disclosure", + mode, + ], + { + artifactName: `onboard-tool-disclosure-${mode}`, + env: hostEnv({ + ...hosted.env, + NEMOCLAW_AGENT: AGENT, + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: name, + NEMOCLAW_TOOL_DISCLOSURE: mode, + }), + redactionValues: [hosted.apiKey], + timeoutMs: 20 * 60_000, + }, + ); + expect(onboard.exitCode, resultText(onboard)).toBe(0); + + const attestation = await sandbox.exec( + name, + ["nemoclaw-start", "sh", "-c", `test "$NEMOCLAW_TOOL_DISCLOSURE" = ${JSON.stringify(mode)}`], + { + artifactName: `attest-tool-disclosure-${mode}`, + env: sandboxAccessEnv(), + timeoutMs: 30_000, + }, + ); + expect(attestation.exitCode, resultText(attestation)).toBe(0); + } + + for (const mode of MODES) { + const add = await host.nemoclaw( + [ + sandboxName(mode), + "mcp", + "add", + MCP_SERVER_NAME, + "--url", + configuredMcpUrl, + "--env", + MCP_TOKEN_ENV, + ], + { + artifactName: `add-tool-disclosure-mcp-${mode}`, + env: hostEnv({ [MCP_TOKEN_ENV]: bearerToken }), + redactionValues: [bearerToken, configuredMcpUrl], + timeoutMs: 5 * 60_000, + }, + ); + expect(add.exitCode, resultText(add)).toBe(0); + + const snapshotProbe = await sandbox.exec( + sandboxName(mode), + ["nemoclaw-start", "/opt/venv/bin/python3", "-I", "-c", MANAGED_MCP_SNAPSHOT_PROBE], + { + artifactName: `managed-mcp-snapshot-${mode}`, + env: sandboxAccessEnv(), + timeoutMs: 30_000, + }, + ); + expect(snapshotProbe.exitCode, resultText(snapshotProbe)).toBe(0); + expect([ + "managed-mcp-snapshot-ok:sealed-memfd", + "managed-mcp-snapshot-ok:anonymous-otmpfile", + ]).toContain(snapshotProbe.stdout.trim()); + } + + const results = []; + for (const mode of MODES) { + const runId = `performance-smoke-${AGENT}-${mode}-${TASK_ID}`; + const driver = buildAgentDriverCommand({ + openshellBin: process.env.OPENSHELL_BIN, + sandboxName: sandboxName(mode), + agent: AGENT, + prompt: frozenTask.prompt, + sessionId: runId, + }); + mcp.beginRun(runId); + const startedAt = process.hrtime.bigint(); + let invocation; + let calls; + try { + invocation = await host.command(driver.command, driver.args, { + artifactName: `invoke-tool-disclosure-${mode}`, + env: hostEnv(), + redactionValues: [...driver.redactions, hosted.apiKey, bearerToken], + timeoutMs: 10 * 60_000, + }); + } finally { + calls = mcp.endRun(); + } + const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); + const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); + const graded = gradeTaskRun(frozenTask, calls, finalOutput); + const outcome = invocation.timedOut + ? "timeout" + : invocation.exitCode === 0 + ? graded.outcome + : "model-error"; + results.push({ + run_id: runId, + mode, + outcome, + invocation: { + exit_code: invocation.exitCode, + timed_out: invocation.timedOut, + elapsed_ms: elapsedMs, + }, + synthetic_call_count: calls.length, + correctness: graded.correctness, + }); + } + + const routedRunId = `performance-smoke-${AGENT}-compositional-${TASK_ID}`; + const routedToolNames = new Set( + catalogPrefix.tools.map((tool) => `${MCP_SERVER_NAME}_${tool.definition.function.name}`), + ); + const expectedRoutedToolName = `${MCP_SERVER_NAME}_${frozenTask.expected_calls[0]?.tool_name ?? ""}`; + expect( + routedToolNames.has(expectedRoutedToolName), + "the routed replay target is not present in the reviewed catalog", + ).toBe(true); + const routedTransform = new CompositionalToolRoutingTransform({ + decomposer: createOpenAIChatTaskDecomposer({ + baseUrl: hosted.endpointUrl, + model: hosted.model, + apiKey: hosted.apiKey, + allowRemote: true, + reasoningControl: "enable_thinking_false", + jsonObjectResponse: true, + maxAttempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, + timeoutMs: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, + }), + embedder: new PortableHashingTextEmbedder(), + isRoutableTool: (name) => routedToolNames.has(name), + requireRouting: true, + }); + const networkInspect = await host.command( + "docker", + ["network", "inspect", "--format", "{{json .IPAM.Config}}", OPENSHELL_DOCKER_NETWORK], + { + artifactName: "compositional-routing-private-network", + env: hostEnv(), + timeoutMs: 30_000, + }, + ); + expect(networkInspect.exitCode, resultText(networkInspect)).toBe(0); + const networkIpam = JSON.parse(networkInspect.stdout) as Array<{ + Gateway?: unknown; + }>; + const privateListenHost = networkIpam + .map((entry) => entry.Gateway) + .find((gateway): gateway is string => typeof gateway === "string" && gateway.includes(".")); + expect(privateListenHost, "OpenShell Docker network has no IPv4 gateway").toBeDefined(); + const recorderListenHost = privateListenHost as string; + const routedProxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: hosted.endpointUrl, + allowRemoteHttpsUpstream: true, + listenHost: recorderListenHost, + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: `Bearer ${routingIngressToken}`, + upstreamAuthorization: `Bearer ${hosted.apiKey}`, + requestTimeoutMs: ROUTED_PROXY_REQUEST_TIMEOUT_MS, + requestTransform: routedTransform.requestTransform, + }); + const routedProxyAddress = await routedProxy.start(); + cleanup.add("stop compositional routing recording proxy", () => routedProxy.stop()); + expect(routedProxyAddress.host).toBe(recorderListenHost); + expect(routedProxyAddress.port).toBeGreaterThanOrEqual(1_024); + + const bridgeProbe = await host.command( + "docker", + [ + "run", + "--rm", + "--pull=missing", + "--network", + OPENSHELL_DOCKER_NETWORK, + PRIVATE_BRIDGE_PROBE_IMAGE, + "nc", + "-zw5", + recorderListenHost, + String(routedProxyAddress.port), + ], + { + artifactName: "compositional-routing-private-reachability", + env: hostEnv(), + redactionValues: [routedProxyAddress.base_url], + timeoutMs: 30_000, + }, + ); + expect( + bridgeProbe.exitCode, + `OpenShell Docker network cannot reach the authenticated routing bridge: ${resultText(bridgeProbe)}`, + ).toBe(0); + + const configureInferenceRoute = async ( + baseUrl: string, + artifactSuffix: string, + credential: string, + timeoutMs: number, + ) => { + artifacts.addRedactionValues([baseUrl, credential]); + const providerEnv = hostEnv({ + [hosted.credentialEnv]: credential, + }); + const update = await host.command( + "openshell", + [ + "provider", + "update", + hosted.providerName, + "--credential", + hosted.credentialEnv, + "--config", + `OPENAI_BASE_URL=${baseUrl}`, + ], + { + artifactName: `compositional-routing-provider-${artifactSuffix}`, + env: providerEnv, + redactionValues: [hosted.apiKey, routingIngressToken, baseUrl, credential], + timeoutMs: 2 * 60_000, + }, + ); + expect(update.exitCode, resultText(update)).toBe(0); + const select = await host.command( + "openshell", + [ + "inference", + "set", + "--no-verify", + "--provider", + hosted.providerName, + "--model", + hosted.model, + "--timeout", + String(timeoutMs / 1_000), + ], + { + artifactName: `compositional-routing-select-${artifactSuffix}`, + env: providerEnv, + redactionValues: [hosted.apiKey, routingIngressToken, baseUrl, credential], + timeoutMs: 2 * 60_000, + }, + ); + expect(select.exitCode, resultText(select)).toBe(0); + }; + + let providerRestore: Promise | undefined; + const restoreInferenceRoute = (): Promise => + providerRestore ?? + (providerRestore = configureInferenceRoute( + hosted.endpointUrl, + "restore", + hosted.apiKey, + RESTORED_GATEWAY_REQUEST_TIMEOUT_MS, + ).catch((error: unknown) => { + providerRestore = undefined; + throw error; + })); + cleanup.add("restore hosted inference route", restoreInferenceRoute); + + let routedReplay: Record | undefined; + try { + await configureInferenceRoute( + `${routedProxyAddress.base_url}/v1`, + "enable", + routingIngressToken, + ROUTED_GATEWAY_REQUEST_TIMEOUT_MS, + ); + const driver = buildAgentDriverCommand({ + openshellBin: process.env.OPENSHELL_BIN, + sandboxName: sandboxName("direct"), + agent: AGENT, + prompt: frozenTask.prompt, + sessionId: routedRunId, + }); + mcp.beginRun(routedRunId); + routedProxy.beginRun(routedRunId); + const startedAt = process.hrtime.bigint(); + let invocation; + let calls; + let recorderEvents; + try { + invocation = await host.command(driver.command, driver.args, { + artifactName: "invoke-tool-disclosure-compositional", + env: hostEnv(), + redactionValues: [ + ...driver.redactions, + hosted.apiKey, + bearerToken, + routingIngressToken, + routedProxyAddress.base_url, + ], + timeoutMs: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, + }); + } finally { + calls = mcp.endRun(); + recorderEvents = routedProxy.endRun(); + } + const elapsedMs = Number((process.hrtime.bigint() - startedAt) / 1_000_000n); + const routeEvidence = await routedTransform.consumeEvidence(routedRunId); + const finalOutput = extractFinalAssistantOutput(AGENT, invocation.stdout); + const graded = gradeTaskRun(frozenTask, calls, finalOutput); + const outcome = invocation.timedOut + ? "timeout" + : invocation.exitCode === 0 + ? graded.outcome + : "model-error"; + routedReplay = { + schema_version: "nemoclaw.compositional_tool_routing_agent_smoke.v1", + generated_at: new Date().toISOString(), + claim_eligible: false, + run_id: routedRunId, + outcome, + invocation: { + exit_code: invocation.exitCode, + timed_out: invocation.timedOut, + elapsed_ms: elapsedMs, + }, + synthetic_call_count: calls.length, + correctness: graded.correctness, + expected_routed_tool_name: expectedRoutedToolName, + routing_configuration: { + reasoning_control: "enable_thinking_false", + output_mode: "json-object", + max_attempts: ROUTING_DECOMPOSER_MAX_ATTEMPTS, + timeout_ms: ROUTING_DECOMPOSER_REQUEST_TIMEOUT_MS, + route_only_run_timeout_ms: ROUTE_ONLY_RUN_TIMEOUT_MS, + proxy_request_timeout_ms: ROUTED_PROXY_REQUEST_TIMEOUT_MS, + gateway_request_timeout_ms: ROUTED_GATEWAY_REQUEST_TIMEOUT_MS, + agent_invocation_timeout_ms: ROUTED_AGENT_INVOCATION_TIMEOUT_MS, + baseline_gateway_request_timeout_ms: RESTORED_GATEWAY_REQUEST_TIMEOUT_MS, + gateway_transport: "authenticated-private-host-bridge", + embedding: "portable-lexical-hashing", + }, + route_evidence: routeEvidence, + recorder_events: recorderEvents, + limitations: [ + "This single routed replay is an end-to-end wiring check, not a performance or quality claim.", + "The portable lexical embedder is used only for an ordinary-runner smoke path.", + "The replay uses an authenticated private runner bridge and replaces its ephemeral ingress credential before the hosted request.", + "The bridge uses host-local HTTP inside the trusted runner and Docker-network boundary.", + "Proxy and gateway timeouts apply per request; the routed agent invocation has its own overall bound.", + "The routed replay is separate from the frozen direct/progressive comparison.", + ], + }; + } finally { + await restoreInferenceRoute(); + } + + await artifacts.writeJson("compositional-routing-agent-smoke.json", routedReplay); + expect(routedReplay).toBeDefined(); + const replay = routedReplay as { + outcome: string; + correctness: { task_success: boolean }; + synthetic_call_count: number; + route_evidence: Awaited>; + recorder_events: Array<{ + model_call_sequence: number | null; + visible_tool_count: number; + }>; + }; + expect(replay.outcome).toBe("success"); + expect(replay.correctness.task_success).toBe(true); + expect(replay.synthetic_call_count).toBe(1); + expect(replay.route_evidence.length).toBeGreaterThan(0); + const [firstRoute] = replay.route_evidence; + expect(firstRoute.routing.fallback).toBeNull(); + expect(firstRoute.transform_bypass).toBeNull(); + expect(firstRoute.routable_tool_count).toBe(CATALOG_SIZE); + expect(firstRoute.forwarded_tool_count).toBeLessThan(firstRoute.source_tool_count); + expect(firstRoute.routing.selected_tool_names).toContain(expectedRoutedToolName); + const firstModelEvent = replay.recorder_events.find((event) => event.model_call_sequence === 1); + expect(firstModelEvent?.visible_tool_count).toBe(firstRoute.forwarded_tool_count); + + await artifacts.writeJson("tool-disclosure-performance-smoke.json", { + schema_version: "nemoclaw.tool_disclosure_performance_smoke.v1", + generated_at: new Date().toISOString(), + claim_eligible: false, + sut_git_sha: gitSha(), + profile: { + agent: AGENT, + modes: MODES, + catalog_seed: catalog.seed, + catalog_size: CATALOG_SIZE, + catalog_tools_sha256: catalogPrefix.tools_sha256, + task_id: frozenTask.id, + task_kind: frozenTask.kind, + task_set_sha256: primaryTasks.tasks_sha256, + repetitions_per_mode: 1, + observed_runs: results.length, + }, + inference: { + api: "openai-compatible", + model_id: hosted.model, + }, + mcp_transport: mcpTransport.kind, + host: { + platform: process.platform, + release: os.release(), + architecture: process.arch, + cpu_model: os.cpus()[0]?.model ?? "unreported", + logical_cpu_count: os.cpus().length, + total_memory_bytes: os.totalmem(), + }, + results, + limitations: [ + "This performance smoke test verifies live wiring and task completion; it is not the complete two-campaign performance test.", + "One observation per mode is insufficient for performance or quality claims.", + "The fixed progressive-then-direct order makes elapsed times informational and subject to cold-cache and ordering effects.", + "The performance smoke test does not collect vLLM tokenizer, token-counter, or request-recorder evidence.", + ...(mcpTransport.kind === "external-ssh-forward" + ? [ + "The MCP HTTPS endpoint is relayed through the CI runner over an authenticated SSH connection to the host-local synthetic server.", + ] + : []), + ], + }); + + expect(results).toHaveLength(2); + for (const result of results) { + expect(result.outcome, `${result.mode} performance smoke outcome`).toBe("success"); + expect(result.correctness.task_success, `${result.mode} task correctness`).toBe(true); + expect(result.synthetic_call_count, `${result.mode} synthetic call count`).toBe(1); + } +}); diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 0bd9353938..a5a1963fab 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -23,7 +23,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { expect(inventory.allowedJobs).toContain("jetson-nvmap-gpu"); expect(inventory.explicitOnlyJobs).toContain("jetson-nvmap-gpu"); expect(formatFreeStandingJobsInventoryForShell(inventory)).toContain( - "explicit_only_jobs_csv=openshell-gateway-auth-contract,mcp-bridge-dev,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", + "explicit_only_jobs_csv=openshell-gateway-auth-contract,mcp-bridge-dev,tool-disclosure-performance-smoke,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", ); expect(inventory.targetToJob.get("jetson-nvmap-gpu")).toBe("jetson-nvmap-gpu"); expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( diff --git a/test/e2e/support/tool-disclosure-performance-smoke-workflow-boundary.test.ts b/test/e2e/support/tool-disclosure-performance-smoke-workflow-boundary.test.ts new file mode 100644 index 0000000000..5682d512db --- /dev/null +++ b/test/e2e/support/tool-disclosure-performance-smoke-workflow-boundary.test.ts @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { UPLOAD_E2E_ARTIFACTS_ACTION } from "../../../tools/e2e/upload-e2e-artifacts-workflow-boundary.mts"; +import { + evaluateE2eWorkflowDispatchSelectors, + readFreeStandingJobsInventory, + validateE2eWorkflowBoundary, +} from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +type WorkflowStep = { + env?: Record; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type WorkflowJob = { + env?: Record; + if?: string; + needs?: string | string[]; + permissions?: Record; + "runs-on"?: string; + steps?: WorkflowStep[]; + "timeout-minutes"?: number; +}; + +function performanceSmokeJob(): WorkflowJob { + const workflow = readWorkflow() as { jobs: Record }; + const job = workflow.jobs["tool-disclosure-performance-smoke"]; + expect(job).toBeDefined(); + return job; +} + +function namedStep(job: WorkflowJob, name: string): WorkflowStep | undefined { + return job.steps?.find((step) => step.name === name); +} + +describe("tool-disclosure performance smoke workflow boundary", () => { + it("keeps the hosted-inference smoke explicit-only", () => { + const inventory = readFreeStandingJobsInventory(); + + expect(validateE2eWorkflowBoundary()).toEqual([]); + expect(inventory.allowedJobs).toContain("tool-disclosure-performance-smoke"); + expect(inventory.explicitOnlyJobs).toContain("tool-disclosure-performance-smoke"); + expect(inventory.targetToJob.get("tool-disclosure-performance-smoke")).toBe( + "tool-disclosure-performance-smoke", + ); + expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( + "tool-disclosure-performance-smoke", + ); + + for (const selector of [ + { jobs: "tool-disclosure-performance-smoke" }, + { targets: "tool-disclosure-performance-smoke" }, + ]) { + expect(evaluateE2eWorkflowDispatchSelectors(selector)).toMatchObject({ + valid: true, + liveTargetsRun: false, + selectedFreeStandingJobs: ["tool-disclosure-performance-smoke"], + registryTargets: [], + }); + } + }); + + it("runs the focused live test with reviewed prerequisites and scoped secrets", () => { + const job = performanceSmokeJob(); + expect(job).toMatchObject({ + needs: "generate-matrix", + if: "${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-performance-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-performance-smoke,') }}", + "runs-on": "ubuntu-latest", + permissions: { contents: "read" }, + "timeout-minutes": 60, + env: { + E2E_JOB: "1", + E2E_DEFAULT_ENABLED: "0", + E2E_TARGET_ID: "tool-disclosure-performance-smoke", + E2E_ARTIFACT_DIR: + "${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-performance-smoke", + NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_OPENSHELL_CHANNEL: "stable", + OPENSHELL_GATEWAY: "nemoclaw", + }, + }); + expect(job.env).not.toHaveProperty("NVIDIA_INFERENCE_API_KEY"); + + const checkout = job.steps?.find((step) => step.uses?.startsWith("actions/checkout@")); + expect(checkout?.with?.["persist-credentials"]).toBe(false); + expect(namedStep(job, "Authenticate to Docker Hub")).toBeDefined(); + expect(namedStep(job, "Prepare E2E workspace")?.uses).toBe( + "NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28", + ); + + const cloudflared = namedStep(job, "Install and verify cloudflared prerequisite"); + expect(cloudflared?.env).toEqual({ + CLOUDFLARED_VERSION: "2026.6.1", + CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526", + }); + expect(cloudflared?.run).toContain("sha256sum -c -"); + expect(cloudflared?.run).toContain("dpkg-deb -f"); + + const run = namedStep(job, "Run tool-disclosure performance smoke live test"); + expect(run?.env).toEqual({ + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + }); + expect(run?.run).toContain("test/e2e/live/tool-disclosure-performance-smoke.test.ts"); + expect(run?.run).toContain("--project e2e-live"); + }); + + it("maps only the smoke artifact directory through the reviewed uploader", () => { + const job = performanceSmokeJob(); + const upload = namedStep(job, "Upload tool-disclosure performance smoke artifacts"); + expect(upload).toEqual({ + name: "Upload tool-disclosure performance smoke artifacts", + if: "always()", + uses: UPLOAD_E2E_ARTIFACTS_ACTION, + }); + expect(job.env?.E2E_TARGET_ID).toBe("tool-disclosure-performance-smoke"); + expect(job.env?.E2E_ARTIFACT_DIR).toBe( + "${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-performance-smoke", + ); + expect(job.steps?.at(-1)?.name).toBe("Clean up Docker auth"); + }); +}); diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 9985da8cf1..c9b6e0f020 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -177,8 +177,8 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "upload-e2e-artifacts must cover exactly 74 live and E2E_JOB execution jobs", - "upload-e2e-artifacts must keep exactly 63 default callers", + "upload-e2e-artifacts must cover exactly 75 live and E2E_JOB execution jobs", + "upload-e2e-artifacts must keep exactly 64 default callers", ]), ); }); diff --git a/test/helpers/langchain-deepagents-code-headless.ts b/test/helpers/langchain-deepagents-code-headless.ts index b11a299403..02f6a2f1e9 100644 --- a/test/helpers/langchain-deepagents-code-headless.ts +++ b/test/helpers/langchain-deepagents-code-headless.ts @@ -50,6 +50,7 @@ export function makeStartScriptFixture( const scriptPath = path.join(tempDir, "start.sh"); const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); + const toolDisclosureFile = path.join(tempDir, "trusted-tool-disclosure"); expect(original).toContain("local target=/tmp/nemoclaw-proxy-env.sh"); expect(original).toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); const fixture = original @@ -61,6 +62,10 @@ export function makeStartScriptFixture( 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', `readonly MANAGED_PROXY_PORT_FILE="${portFile}"`, ) + .replace( + 'readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure"', + `readonly MANAGED_TOOL_DISCLOSURE_FILE="${toolDisclosureFile}"`, + ) .replace( "readonly MANAGED_PROXY_OWNER_UID=0", `readonly MANAGED_PROXY_OWNER_UID=${process.getuid?.() ?? 0}`, @@ -76,8 +81,10 @@ export function makeStartScriptFixture( expect(fixture).not.toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); fs.writeFileSync(hostFile, "10.200.0.1\n", "utf8"); fs.writeFileSync(portFile, "3128\n", "utf8"); + fs.writeFileSync(toolDisclosureFile, "progressive\n", "utf8"); fs.chmodSync(hostFile, 0o444); fs.chmodSync(portFile, 0o444); + fs.chmodSync(toolDisclosureFile, 0o444); fs.writeFileSync(scriptPath, fixture, "utf8"); fs.chmodSync(scriptPath, 0o755); return { envFile, scriptPath }; diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index 1a157bf727..e903fa5387 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -422,12 +422,15 @@ describe("LangChain Deep Agents Code image contracts", () => { it("keeps optional service egress out of the default policy and requires Landlock", () => { const policy = readAgentFile("policy-additions.yaml"); + const managedRuntime = readAgentFile("managed-dcode-runtime.py"); expect(policy).not.toContain("api.tavily.com"); expect(policy).not.toContain("api.smith.langchain.com"); expect(policy).toContain(" - /usr\n"); expect(policy).toContain(" - /opt/venv\n"); expect(policy).toContain(" - /etc\n"); + expect(policy).toContain(" - /dev/shm\n"); + expect(managedRuntime).toContain('(Path("/tmp"), Path("/dev/shm"))'); expect(policy).toContain("compatibility: strict"); expect(policy).not.toContain("compatibility: best_effort"); expect(policy).toContain("fail closed when Landlock cannot be applied"); diff --git a/test/langchain-deepagents-code-managed-mcp-hardening.test.ts b/test/langchain-deepagents-code-managed-mcp-hardening.test.ts index 03a5202112..c6313a6916 100644 --- a/test/langchain-deepagents-code-managed-mcp-hardening.test.ts +++ b/test/langchain-deepagents-code-managed-mcp-hardening.test.ts @@ -136,10 +136,21 @@ assert payload is not None def blocked_memfd(*_args, **_kwargs): raise PermissionError(errno.EPERM, "blocked by seccomp") +real_open = managed.os.open +anonymous_attempts = [] + +def force_tmpfile_to_shm(path, flags, *args, **kwargs): + if flags & os.O_TMPFILE: + anonymous_attempts.append(str(path)) + if str(path) == "/tmp": + raise OSError(errno.EOPNOTSUPP, "O_TMPFILE unavailable") + return real_open(path, flags, *args, **kwargs) + with tempfile.TemporaryDirectory() as tempdir: managed._MCP_CONFIG_FILE = Path(tempdir) / ".nemoclaw-mcp.json" managed._read_managed_mcp_config = lambda: raw managed.os.memfd_create = blocked_memfd + managed.os.open = force_tmpfile_to_shm snapshot_path = managed.managed_mcp_config_path() assert snapshot_path is not None descriptor = int(snapshot_path.removeprefix("/proc/self/fd/")) @@ -220,6 +231,8 @@ print(child.managed_mcp_config_bytes(sys.argv[2]).decode(), end="") else: raise AssertionError("same-size anonymous descriptor overwrite was accepted") os.close(descriptor) +managed.os.open = real_open +assert anonymous_attempts == ["/tmp", "/dev/shm", "/tmp", "/dev/shm"] print("anonymous-fallback-ok") `); @@ -332,4 +345,81 @@ print("fallback-fail-closed-ok") expect(result.status, result.stderr).toBe(0); expect(result.stdout.trim()).toBe("fallback-fail-closed-ok"); }); + + it("tries the bounded anonymous tmpfs fallback only for unsupported filesystems", () => { + const result = runManagedHelper(String.raw` +import errno +import importlib.util +import os +import sys +from pathlib import Path + +spec = importlib.util.spec_from_file_location("_nemoclaw_managed", sys.argv[1]) +managed = importlib.util.module_from_spec(spec) +spec.loader.exec_module(managed) + +managed._MCP_ANONYMOUS_DIRECTORIES = (Path("/tmp"), Path("/dev/shm")) +real_open = managed.os.open + +for fallback_errno in managed._MCP_ANONYMOUS_DIRECTORY_FALLBACK_ERRNOS: + attempts = [] + + def filesystem_fallback(path, flags, *args, **kwargs): + attempts.append(str(path)) + if str(path) == "/tmp": + raise OSError(fallback_errno, "anonymous file unavailable") + return 123 + + managed.os.open = filesystem_fallback + try: + writer = managed._open_anonymous_managed_mcp_writer(12345) + assert writer == 123 + assert attempts == ["/tmp", "/dev/shm"] + finally: + managed.os.open = real_open + +for terminal_errno in (errno.EACCES, errno.EPERM, errno.ENOSPC, errno.EMFILE): + attempts = [] + + def terminal_error(path, flags, *args, **kwargs): + attempts.append(str(path)) + raise OSError(terminal_errno, "terminal anonymous-file error") + + managed.os.open = terminal_error + try: + try: + managed._open_anonymous_managed_mcp_writer(12345) + except OSError as exc: + assert exc.errno == terminal_errno + else: + raise AssertionError("terminal error triggered a directory fallback") + assert attempts == ["/tmp"] + finally: + managed.os.open = real_open + +attempts = [] + +def second_directory_error(path, flags, *args, **kwargs): + attempts.append(str(path)) + if str(path) == "/tmp": + raise OSError(errno.EOPNOTSUPP, "O_TMPFILE unavailable") + raise OSError(errno.ENOSPC, "tmpfs full") + +managed.os.open = second_directory_error +try: + try: + managed._open_anonymous_managed_mcp_writer(12345) + except OSError as exc: + assert exc.errno == errno.ENOSPC + else: + raise AssertionError("final directory error was masked") + assert attempts == ["/tmp", "/dev/shm"] +finally: + managed.os.open = real_open +print("anonymous-directory-fallback-ok") +`); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim()).toBe("anonymous-directory-fallback-ok"); + }); }); diff --git a/test/langchain-deepagents-code-proxy-launcher.test.ts b/test/langchain-deepagents-code-proxy-launcher.test.ts index ad20559834..096cdceee4 100644 --- a/test/langchain-deepagents-code-proxy-launcher.test.ts +++ b/test/langchain-deepagents-code-proxy-launcher.test.ts @@ -31,15 +31,20 @@ function readAgentFile(name: string): string { function writeManagedProxyFiles( tempDir: string, managedProxy: { host: string; port: string }, + toolDisclosure: "progressive" | "direct" = "progressive", ): void { const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); + const toolDisclosureFile = path.join(tempDir, "trusted-tool-disclosure"); fs.rmSync(hostFile, { force: true }); fs.rmSync(portFile, { force: true }); + fs.rmSync(toolDisclosureFile, { force: true }); fs.writeFileSync(hostFile, `${managedProxy.host}\n`); fs.writeFileSync(portFile, `${managedProxy.port}\n`); + fs.writeFileSync(toolDisclosureFile, `${toolDisclosure}\n`); fs.chmodSync(hostFile, 0o444); fs.chmodSync(portFile, 0o444); + fs.chmodSync(toolDisclosureFile, 0o444); } function replaceManagedProxyFileConstants(source: string, tempDir: string): string { @@ -52,6 +57,10 @@ function replaceManagedProxyFileConstants(source: string, tempDir: string): stri 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', `readonly MANAGED_PROXY_PORT_FILE="${path.join(tempDir, "trusted-proxy-port")}"`, ) + .replace( + 'readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure"', + `readonly MANAGED_TOOL_DISCLOSURE_FILE="${path.join(tempDir, "trusted-tool-disclosure")}"`, + ) .replace( "readonly MANAGED_PROXY_OWNER_UID=0", `readonly MANAGED_PROXY_OWNER_UID=${TEST_OWNER_UID}`, @@ -61,12 +70,13 @@ function replaceManagedProxyFileConstants(source: string, tempDir: string): stri function makeLauncherProxyProbeFixture( tempDir: string, managedProxy: { host: string; port: string } = DEFAULT_MANAGED_PROXY, + toolDisclosure: "progressive" | "direct" = "progressive", ): string { const launcherPath = path.join(tempDir, "dcode-launcher.sh"); const probePath = path.join(tempDir, "managed-dcode-probe.sh"); const probe = [ "#!/bin/bash -p", - "for name in HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ALL_PROXY all_proxy OPENAI_PROXY NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT; do", + "for name in HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ALL_PROXY all_proxy OPENAI_PROXY NEMOCLAW_PROXY_HOST NEMOCLAW_PROXY_PORT NEMOCLAW_TOOL_DISCLOSURE; do", ' printf \'LAUNCHER_%s=%s\\n\' "$name" "${!name-__unset__}"', "done", "", @@ -80,7 +90,7 @@ function makeLauncherProxyProbeFixture( ); fs.writeFileSync(probePath, probe, "utf8"); fs.writeFileSync(launcherPath, fixture, "utf8"); - writeManagedProxyFiles(tempDir, managedProxy); + writeManagedProxyFiles(tempDir, managedProxy, toolDisclosure); fs.chmodSync(probePath, 0o755); fs.chmodSync(launcherPath, 0o755); return launcherPath; @@ -89,6 +99,7 @@ function makeLauncherProxyProbeFixture( function makeStartProxyProbeFixture( tempDir: string, managedProxy: { host: string; port: string } = DEFAULT_MANAGED_PROXY, + toolDisclosure: "progressive" | "direct" = "progressive", ): { envFile: string; scriptPath: string } { const envFile = path.join(tempDir, "proxy-env.sh"); const scriptPath = path.join(tempDir, "start.sh"); @@ -99,7 +110,7 @@ function makeStartProxyProbeFixture( `tmp="$(mktemp "${tempDir}/nemoclaw-proxy-env.XXXXXX")"`, ); fs.writeFileSync(scriptPath, fixture, "utf8"); - writeManagedProxyFiles(tempDir, managedProxy); + writeManagedProxyFiles(tempDir, managedProxy, toolDisclosure); fs.chmodSync(scriptPath, 0o755); return { envFile, scriptPath }; } @@ -198,6 +209,37 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { expect(output).not.toContain("all-password"); }); + it("restores the image-selected tool-disclosure mode on raw OpenShell exec paths", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-disclosure-")); + const launcherPath = makeLauncherProxyProbeFixture(tempDir, DEFAULT_MANAGED_PROXY, "direct"); + const { envFile, scriptPath } = makeStartProxyProbeFixture( + tempDir, + DEFAULT_MANAGED_PROXY, + "direct", + ); + const untrustedEnv = { NEMOCLAW_TOOL_DISCLOSURE: "progressive" }; + const launcherResult = runLauncher(launcherPath, ["-n", "PONG"], untrustedEnv); + const startResult = spawnSync( + "bash", + [ + scriptPath, + "bash", + "-c", + 'printf "START_TOOL_DISCLOSURE=%s\\n" "$NEMOCLAW_TOOL_DISCLOSURE"', + ], + { + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", ...untrustedEnv }, + encoding: "utf8", + }, + ); + + expect(launcherResult.status, launcherResult.stderr).toBe(0); + expect(startResult.status, startResult.stderr).toBe(0); + expect(launcherResult.stdout).toContain("LAUNCHER_NEMOCLAW_TOOL_DISCLOSURE=direct"); + expect(startResult.stdout).toContain("START_TOOL_DISCLOSURE=direct"); + expect(fs.readFileSync(envFile, "utf8")).toContain("export NEMOCLAW_TOOL_DISCLOSURE=direct"); + }); + it("pins validated proxy overrides into direct dcode execution paths (#6191)", () => { const dockerfile = readAgentFile("Dockerfile"); const launcher = readAgentFile("dcode-launcher.sh"); @@ -208,6 +250,11 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { expect(dockerfile).toContain("printf '%s\\n' \"$NEMOCLAW_PROXY_PORT\""); expect(dockerfile).toContain("chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host"); expect(dockerfile).toContain("chown root:root /usr/local/share/nemoclaw/dcode-proxy-host"); + expect(dockerfile).toContain( + "printf '%s\\n' \"$NEMOCLAW_TOOL_DISCLOSURE\" > /usr/local/share/nemoclaw/dcode-tool-disclosure", + ); + expect(dockerfile).toMatch(/chmod 0444 .*dcode-tool-disclosure/u); + expect(dockerfile).toMatch(/chown root:root .*dcode-tool-disclosure/u); expect(dockerfile).not.toContain(" NEMOCLAW_PROXY_HOST=${NEMOCLAW_PROXY_HOST}"); expect(dockerfile).not.toContain(" NEMOCLAW_PROXY_PORT=${NEMOCLAW_PROXY_PORT}"); expect(launcher).toContain('readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw'); @@ -325,6 +372,34 @@ describe("Deep Agents Code direct-exec proxy launcher", () => { ); }); + it("fails closed on missing or invalid image-baked tool-disclosure state", () => { + for (const invalidValue of [null, "unexpected"] as const) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-disclosure-invalid-")); + const launcherPath = makeLauncherProxyProbeFixture(tempDir); + const { scriptPath } = makeStartProxyProbeFixture(tempDir); + const modeFile = path.join(tempDir, "trusted-tool-disclosure"); + fs.rmSync(modeFile); + invalidValue === null + ? undefined + : fs.writeFileSync(modeFile, `${invalidValue}\n`, { mode: 0o444 }); + + const env = { NEMOCLAW_TOOL_DISCLOSURE: "direct" }; + const launcherResult = runLauncher(launcherPath, ["-n", "PONG"], env); + const startResult = spawnSync("bash", [scriptPath, "true"], { + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", ...env }, + encoding: "utf8", + }); + + expect(launcherResult.status).not.toBe(0); + expect(startResult.status).not.toBe(0); + const combined = `${launcherResult.stdout}\n${launcherResult.stderr}\n${startResult.stdout}\n${startResult.stderr}`; + expect(combined).toMatch( + /trusted managed tool-disclosure mode file|Invalid image-baked tool-disclosure mode/, + ); + expect(combined).not.toContain("LAUNCHER_NEMOCLAW_TOOL_DISCLOSURE=direct"); + } + }); + it("keeps dcode shell proxy validators aligned with onboard validation (#6191)", () => { const start = readAgentFile("start.sh"); const launcher = readAgentFile("dcode-launcher.sh"); diff --git a/test/performance/tool-disclosure-catalog.test.ts b/test/performance/tool-disclosure-catalog.test.ts new file mode 100644 index 0000000000..0a23116e3b --- /dev/null +++ b/test/performance/tool-disclosure-catalog.test.ts @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeAll, describe, expect, it } from "vitest"; + +import { + assertValidSyntheticCatalog, + buildSyntheticArguments, + canonicalJson, + DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED, + executeSyntheticTool, + generateCatalogPrefix, + generateCatalogPrefixes, + generateSyntheticCatalog, + MAX_SYNTHETIC_TOOLS, + SYNTHETIC_CATEGORIES, + type SyntheticCatalog, + SyntheticToolInvocationError, + sha256Hex, + toOpenAIFunctionTools, + validateSyntheticArguments, + validateSyntheticCatalog, +} from "../../scripts/performance/tool-disclosure/catalog"; +import { + assertValidSyntheticTaskSet, + buildPrimaryTasks, + buildStressTasks, + generatePrimaryTaskSet, + generateStressTaskSet, + PRIMARY_CATALOG_MIN_SIZE, + PRIMARY_TASK_COUNT, + STRESS_CATALOG_SIZE, + STRESS_TASK_COUNT, + validateSyntheticTaskSet, +} from "../../scripts/performance/tool-disclosure/tasks"; + +let fullCatalog: SyntheticCatalog; + +beforeAll(() => { + fullCatalog = generateSyntheticCatalog(); +}); + +describe("synthetic tool-disclosure catalog", () => { + it("documents exactly 24 distinct synthetic categories", () => { + expect(SYNTHETIC_CATEGORIES).toHaveLength(24); + expect(new Set(SYNTHETIC_CATEGORIES.map((category) => category.id)).size).toBe(24); + for (const category of SYNTHETIC_CATEGORIES) { + expect(category.label.length).toBeGreaterThan(0); + expect(category.resource.length).toBeGreaterThan(0); + expect(category.purpose.length).toBeGreaterThan(10); + expect(category.operations).toHaveLength(6); + expect(new Set(category.operations).size).toBe(6); + } + }); + + it("generates 2,209 unique, safe OpenAI-style function definitions", () => { + expect(fullCatalog.seed).toBe(DEFAULT_SYNTHETIC_PERFORMANCE_TEST_CATALOG_SEED); + expect(fullCatalog.size).toBe(MAX_SYNTHETIC_TOOLS); + expect(fullCatalog.tools).toHaveLength(2_209); + expect(validateSyntheticCatalog(fullCatalog)).toEqual([]); + expect(() => assertValidSyntheticCatalog(fullCatalog)).not.toThrow(); + + const definitions = toOpenAIFunctionTools(fullCatalog); + const names = definitions.map((definition) => definition.function.name); + expect(definitions).toHaveLength(2_209); + expect(new Set(names).size).toBe(2_209); + expect(names.every((name) => /^[A-Za-z0-9_-]{1,64}$/.test(name))).toBe(true); + expect(definitions.every((definition) => definition.type === "function")).toBe(true); + expect( + definitions.every( + (definition) => + definition.function.description.length > 40 && + definition.function.parameters.type === "object" && + definition.function.parameters.additionalProperties === false, + ), + ).toBe(true); + expect(new Set(fullCatalog.tools.map((tool) => tool.category))).toEqual( + new Set(SYNTHETIC_CATEGORIES.map((category) => category.id)), + ); + }); + + it("uses a stable 25/50/25 schema-complexity cycle with a medium-schema median", () => { + const counts = { small: 0, medium: 0, large: 0 }; + for (const tool of fullCatalog.tools) counts[tool.complexity] += 1; + expect(counts).toEqual({ small: 555, medium: 1_104, large: 550 }); + + const samples = [ + fullCatalog.tools.find((tool) => tool.complexity === "small"), + fullCatalog.tools.find((tool) => tool.complexity === "medium"), + fullCatalog.tools.find((tool) => tool.complexity === "large"), + ]; + for (const tool of samples) { + expect(tool).toBeDefined(); + const requiredTool = tool as NonNullable; + const args = buildSyntheticArguments(requiredTool, 17); + expect(validateSyntheticArguments(requiredTool, args)).toEqual([]); + } + expect(Object.keys(samples[0]?.definition.function.parameters.properties ?? {})).toHaveLength( + 2, + ); + expect(Object.keys(samples[1]?.definition.function.parameters.properties ?? {})).toHaveLength( + 5, + ); + expect(Object.keys(samples[2]?.definition.function.parameters.properties ?? {})).toHaveLength( + 8, + ); + }); + + it("is deterministic, seed-sensitive, serializable, and prefix-stable", () => { + const direct64 = generateSyntheticCatalog({ size: 64 }); + const prefix64 = generateCatalogPrefix(fullCatalog, 64); + expect(prefix64).toEqual(direct64); + expect(generateSyntheticCatalog({ size: 64 })).toEqual(direct64); + expect(generateSyntheticCatalog({ size: 64, seed: 41 })).not.toEqual(direct64); + + const requested = generateCatalogPrefixes(fullCatalog, [16, 64, 256, 512, 2_209]); + expect(requested.map((catalog) => catalog.size)).toEqual([16, 64, 256, 512, 2_209]); + for (const catalog of requested) expect(validateSyntheticCatalog(catalog)).toEqual([]); + expect(JSON.parse(JSON.stringify(direct64))).toEqual(direct64); + }); + + it("rejects invalid sizes, seeds, and corrupted catalog metadata", () => { + expect(() => generateSyntheticCatalog({ size: 0 })).toThrow(/catalog size/); + expect(() => generateSyntheticCatalog({ size: 2_210 })).toThrow(/catalog size/); + expect(() => generateSyntheticCatalog({ seed: "" })).toThrow(/seed/); + expect(() => generateSyntheticCatalog({ seed: -1 })).toThrow(/seed/); + expect(() => generateCatalogPrefix(generateSyntheticCatalog({ size: 16 }), 17)).toThrow( + /exceeds/, + ); + expect(validateSyntheticCatalog({ ...fullCatalog, size: 2_208 })).toContain( + "catalog size does not match tool count", + ); + expect(validateSyntheticCatalog({ ...fullCatalog, tools_sha256: "0".repeat(64) })).toContain( + "catalog tools_sha256 does not match tools", + ); + }); +}); + +describe("canonical fixture execution", () => { + it("canonicalizes object keys and produces a stable SHA-256 digest", () => { + const first = { z: [3, { b: true, a: null }], a: "value" }; + const second = { a: "value", z: [3, { a: null, b: true }] }; + expect(canonicalJson(first)).toBe(canonicalJson(second)); + expect(canonicalJson(first)).toBe('{"a":"value","z":[3,{"a":null,"b":true}]}'); + expect(sha256Hex(canonicalJson(first))).toMatch(/^[a-f0-9]{64}$/); + expect(() => canonicalJson(Number.NaN)).toThrow(/non-finite/); + }); + + it("returns deterministic argument-bound nonces from every executable fixture", () => { + for (const tool of fullCatalog.tools) { + const args = buildSyntheticArguments(tool, tool.index % 503); + expect(validateSyntheticArguments(tool, args)).toEqual([]); + const result = executeSyntheticTool(tool, args); + expect(result.ok).toBe(true); + expect(result.tool_name).toBe(tool.definition.function.name); + expect(result.nonce).toMatch(/^nonce_[a-f0-9]{20}$/); + } + + const tool = fullCatalog.tools[17]; + const args = buildSyntheticArguments(tool, 8); + const reordered = Object.fromEntries(Object.entries(args).reverse()); + expect(executeSyntheticTool(tool, reordered).nonce).toBe( + executeSyntheticTool(tool, args).nonce, + ); + expect(executeSyntheticTool(tool, { ...args, query: "a changed request" }).nonce).not.toBe( + executeSyntheticTool(tool, args).nonce, + ); + }); + + it("rejects malformed handler arguments before producing a result", () => { + const tool = fullCatalog.tools.find((candidate) => candidate.complexity === "large"); + expect(tool).toBeDefined(); + const requiredTool = tool as NonNullable; + const valid = buildSyntheticArguments(requiredTool, 3); + expect(validateSyntheticArguments(requiredTool, { ...valid, unexpected: true })).toContain( + "arguments.unexpected is not allowed", + ); + expect(() => executeSyntheticTool(requiredTool, { ...valid, limit: 0 })).toThrow( + SyntheticToolInvocationError, + ); + }); +}); + +describe("tool-disclosure task fixtures", () => { + it("builds the exact 24-task primary composition with valid oracles", () => { + const taskSet = generatePrimaryTaskSet(fullCatalog); + expect(taskSet.task_count).toBe(PRIMARY_TASK_COUNT); + expect(taskSet.tasks).toHaveLength(24); + expect(validateSyntheticTaskSet(taskSet, fullCatalog)).toEqual([]); + expect(() => assertValidSyntheticTaskSet(taskSet, fullCatalog)).not.toThrow(); + + const count = (kind: string) => taskSet.tasks.filter((task) => task.kind === kind).length; + expect(count("single-tool")).toBe(8); + expect(count("ordered-chain")).toBe(8); + expect(count("near-match")).toBe(4); + expect(count("no-tool")).toBe(4); + expect(new Set(taskSet.tasks.map((task) => task.id)).size).toBe(24); + expect(JSON.parse(JSON.stringify(taskSet))).toEqual(taskSet); + }); + + it("makes every chain consume the first result and every distractor genuinely nearby", () => { + const tasks = buildPrimaryTasks(fullCatalog); + const toolsByName = new Map( + fullCatalog.tools.map((tool) => [tool.definition.function.name, tool]), + ); + + for (const task of tasks.filter((candidate) => candidate.kind === "ordered-chain")) { + expect(task.expected_calls).toHaveLength(2); + expect(task.expected_calls[1].arguments.resource_id).toBe( + task.expected_calls[0].result_nonce, + ); + expect(task.expected_final_includes).toEqual( + task.expected_calls.map((call) => call.result_nonce), + ); + } + for (const task of tasks.filter((candidate) => candidate.kind === "near-match")) { + const target = toolsByName.get(task.expected_calls[0].tool_name); + expect(target).toBeDefined(); + expect(task.distractor_tool_names).toHaveLength(2); + for (const distractorName of task.distractor_tool_names) { + const distractor = toolsByName.get(distractorName); + expect(distractor?.category).toBe(target?.category); + expect(distractor?.operation).not.toBe(target?.operation); + expect(task.expected_calls.map((call) => call.tool_name)).not.toContain(distractorName); + } + } + for (const task of tasks.filter((candidate) => candidate.kind === "no-tool")) { + expect(task.expected_calls).toEqual([]); + expect(task.expected_final_includes).toHaveLength(1); + } + }); + + it("builds eight single-tool stress tasks distributed through the 2,209-tool catalog", () => { + const taskSet = generateStressTaskSet(fullCatalog); + expect(taskSet.task_count).toBe(STRESS_TASK_COUNT); + expect(taskSet.tasks).toHaveLength(8); + expect(taskSet.tasks.every((task) => task.kind === "single-tool")).toBe(true); + expect(taskSet.tasks.map((task) => task.expected_calls[0].tool_name)).toEqual( + [256, 511, 767, 1_023, 1_279, 1_535, 1_791, 2_208].map( + (index) => fullCatalog.tools[index].definition.function.name, + ), + ); + expect(validateSyntheticTaskSet(taskSet, fullCatalog)).toEqual([]); + expect(buildStressTasks(fullCatalog)).toEqual(taskSet.tasks); + }); + + it("keeps primary tasks prefix-invariant and rejects undersized task catalogs", () => { + const prefix = generateCatalogPrefix(fullCatalog, PRIMARY_CATALOG_MIN_SIZE); + expect(generatePrimaryTaskSet(prefix).tasks).toEqual(generatePrimaryTaskSet(fullCatalog).tasks); + expect(() => buildPrimaryTasks(generateSyntheticCatalog({ size: 63 }))).toThrow(/at least 64/); + expect(() => buildStressTasks(generateSyntheticCatalog({ size: 512 }))).toThrow(/require 2209/); + + const valid = generatePrimaryTaskSet(prefix); + expect(validateSyntheticTaskSet({ ...valid, tasks_sha256: "f".repeat(64) }, prefix)).toContain( + "tasks_sha256 does not match tasks", + ); + }); + + it("exposes fixed size constants matching the generated task requirements", () => { + expect(PRIMARY_CATALOG_MIN_SIZE).toBe(64); + expect(STRESS_CATALOG_SIZE).toBe(MAX_SYNTHETIC_TOOLS); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-router.test.ts b/test/performance/tool-disclosure-compositional-router.test.ts new file mode 100644 index 0000000000..3a2c961bf7 --- /dev/null +++ b/test/performance/tool-disclosure-compositional-router.test.ts @@ -0,0 +1,414 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + buildDenseToolIndex, + type DecompositionRequest, + exactInnerProductTopK, + l2Normalize, + routeCompositionalTools, + routeCompositionalToolsPaired, + selectRoutedToolDefinitions, + type TaskDecomposer, + type TextEmbedder, + type ToolRoutingCatalogEntry, + unionToolHints, +} from "../../scripts/performance/tool-disclosure/compositional-tool-router"; + +interface TestDefinition { + type: "function"; + function: { + name: string; + parameters: { type: "object"; properties: Record }; + }; +} + +function definition(name: string): TestDefinition { + return { + type: "function", + function: { + name, + parameters: { type: "object", properties: { value: { type: "string" } } }, + }, + }; +} + +function catalogEntry(name: string, description: string): ToolRoutingCatalogEntry { + return { name, description, definition: definition(name) }; +} + +class KeywordEmbedder implements TextEmbedder { + readonly calls: string[][] = []; + + async embed(texts: readonly string[]): Promise { + this.calls.push([...texts]); + return texts.map((text) => { + const normalized = text.toLowerCase(); + return /calendar|appointment|event/u.test(normalized) + ? [3, 0, 0, 0] + : /email|notify|message/u.test(normalized) + ? [0, 4, 0, 0] + : /weather|forecast/u.test(normalized) + ? [0, 0, 5, 0] + : [0, 0, 0, 2]; + }); + } +} + +describe("independent compositional tool dense retrieval", () => { + it("normalizes vectors and applies exact top-k with score/name tie-breaking", async () => { + expect(l2Normalize([3, 4])).toEqual([0.6, 0.8]); + expect(() => l2Normalize([0, 0])).toThrow("positive finite L2 norm"); + expect(() => l2Normalize([1, Number.NaN])).toThrow("must be finite"); + + const catalog = [ + catalogEntry("zeta", "first horizontal tool"), + catalogEntry("alpha", "second horizontal tool"), + catalogEntry("vertical", "vertical tool"), + ]; + const embedder: TextEmbedder = { + async embed() { + return [ + [20, 0], + [2, 0], + [0, 7], + ]; + }, + }; + const index = await buildDenseToolIndex(catalog, embedder); + + expect(index.tools.map((tool) => tool.vector)).toEqual([ + [1, 0], + [1, 0], + [0, 1], + ]); + expect(exactInnerProductTopK(index, [9, 0], 2)).toEqual([ + { name: "alpha", score: 1 }, + { name: "zeta", score: 1 }, + ]); + + const tied = { + dimension: 1, + tools: Array.from({ length: 20 }, (_, toolIndex) => ({ + name: `tool-${String(toolIndex).padStart(2, "0")}`, + vector: [1], + })), + }; + expect(exactInnerProductTopK(tied, [1])).toHaveLength(10); + }); + + it("unions hints by maximum score before applying the deterministic cap", () => { + const hints = unionToolHints( + [ + [ + { name: "shared", score: 0.2 }, + { name: "bravo", score: 0.8 }, + { name: "charlie", score: 0.7 }, + ], + [ + { name: "shared", score: 0.9 }, + { name: "alpha", score: 0.8 }, + ], + ], + 3, + ); + + expect(hints).toEqual([ + { name: "shared", score: 0.9 }, + { name: "alpha", score: 0.8 }, + { name: "bravo", score: 0.8 }, + ]); + + const defaultHints = unionToolHints([ + Array.from({ length: 10 }, (_, index) => ({ name: `left-${index}`, score: 1 - index / 100 })), + Array.from({ length: 10 }, (_, index) => ({ + name: `right-${index}`, + score: 0.5 - index / 100, + })), + ]); + expect(defaultHints).toHaveLength(12); + }); +}); + +describe("independent compositional tool routing", () => { + it("shares one initial decomposition and one prepared index across paired routes", async () => { + const catalog = [ + catalogEntry("calendar_find", "Find an appointment in a calendar."), + catalogEntry("email_send", "Send an email message to notify a teammate."), + ]; + const decompose = vi.fn(async (request) => + request.pass === "initial" + ? ["locate the appointment", "notify the teammate"] + : ["find the calendar event", "send the email message"], + ); + const embedder = new KeywordEmbedder(); + + const pair = await routeCompositionalToolsPaired("Arrange and notify.", catalog, { + decomposer: { decompose }, + embedder, + }); + + expect(pair.shared_initial_decomposition).toBe(true); + expect(pair.shared_initial_subtask_count).toBe(2); + expect(decompose.mock.calls.filter(([request]) => request.pass === "initial")).toHaveLength(1); + expect(decompose.mock.calls.filter(([request]) => request.pass === "refined")).toHaveLength(1); + expect( + embedder.calls.filter((batch) => batch.every((text) => text.includes("\n"))), + ).toHaveLength(1); + expect(embedder.calls).toHaveLength(3); + expect(embedder.calls[1]).toEqual(["locate the appointment", "notify the teammate"]); + expect(embedder.calls[2]).toEqual(["find the calendar event", "send the email message"]); + expect(pair.initial.evidence.initial_candidate_tool_names).toEqual( + pair.refined.evidence.initial_candidate_tool_names, + ); + expect(pair.initial.evidence.timings.initial_decomposition_ms).toBe( + pair.refined.evidence.timings.initial_decomposition_ms, + ); + expect(pair.initial.evidence.timings.catalog_embedding_ms).toBe( + pair.refined.evidence.timings.catalog_embedding_ms, + ); + expect(pair.initial.evidence.timings.initial_retrieval_ms).toBe( + pair.refined.evidence.timings.initial_retrieval_ms, + ); + expect(pair.initial.selected_tool_names).toEqual(pair.refined.selected_tool_names); + }); + + it("shares paired initial failures without starting retrieval or refinement", async () => { + const decompose = vi.fn().mockResolvedValue({ malformed: true }); + const embed = vi.fn(); + const pair = await routeCompositionalToolsPaired( + "Arrange and notify.", + [catalogEntry("calendar_find", "Find a calendar event.")], + { decomposer: { decompose }, embedder: { embed } }, + ); + + expect(decompose).toHaveBeenCalledOnce(); + expect(embed).not.toHaveBeenCalled(); + expect(pair.initial).toMatchObject({ + disposition: "passthrough", + evidence: { fallback: "initial-decomposition-malformed" }, + }); + expect(pair.refined).toMatchObject({ + disposition: "passthrough", + evidence: { fallback: "initial-decomposition-malformed" }, + }); + expect(pair.initial.evidence.timings.initial_decomposition_ms).toBe( + pair.refined.evidence.timings.initial_decomposition_ms, + ); + }); + + it("shares a paired no-tool result without embedding or refining", async () => { + const decompose = vi.fn().mockResolvedValue([]); + const embed = vi.fn(); + const pair = await routeCompositionalToolsPaired( + "Answer without tools.", + [catalogEntry("calendar_find", "Find a calendar event.")], + { decomposer: { decompose }, embedder: { embed } }, + ); + + expect(decompose).toHaveBeenCalledOnce(); + expect(embed).not.toHaveBeenCalled(); + expect(pair.initial).toMatchObject({ + disposition: "routed", + selected_tool_names: [], + evidence: { decomposition_passes: 1, fallback: null }, + }); + expect(pair.refined).toMatchObject({ + disposition: "routed", + selected_tool_names: [], + evidence: { decomposition_passes: 1, fallback: null }, + }); + }); + + it("supports a one-pass initial route over the same retriever", async () => { + const catalog = [ + catalogEntry("calendar_find", "Find an appointment in a calendar."), + catalogEntry("email_send", "Send an email message to notify a teammate."), + catalogEntry("weather_read", "Read a weather forecast."), + ]; + const decompose = vi + .fn() + .mockResolvedValue(["find the calendar event", "send the email message"]); + + const result = await routeCompositionalTools("Arrange and notify.", catalog, { + decomposer: { decompose }, + embedder: new KeywordEmbedder(), + refinementPasses: 0, + }); + + expect(decompose).toHaveBeenCalledOnce(); + expect(result.disposition).toBe("routed"); + expect(result.selected_tool_names).toEqual(["calendar_find", "email_send"]); + expect(result.evidence).toMatchObject({ + initial_subtask_count: 2, + refined_subtask_count: 0, + decomposition_passes: 1, + hint_count: 0, + selected_tool_count: 2, + fallback: null, + }); + expect(result.evidence.final_candidate_counts).toEqual([3, 3]); + }); + + it("runs one initial pass and one hint-informed refinement before bounded selection", async () => { + const catalog = [ + catalogEntry("calendar_find", "Find an appointment in a calendar."), + catalogEntry("email_send", "Send an email message to notify a teammate."), + catalogEntry("weather_read", "Read a weather forecast."), + catalogEntry("generic_lookup", "Look up an unrelated record."), + ]; + const requests: DecompositionRequest[] = []; + const decomposer: TaskDecomposer = { + async decompose(request) { + requests.push(request); + return request.pass === "initial" + ? ["locate the appointment", "notify the teammate"] + : ["find the calendar event", "send the email message"]; + }, + }; + const embedder = new KeywordEmbedder(); + const result = await routeCompositionalTools( + "Arrange the appointment, then tell the teammate.", + catalog, + { decomposer, embedder, maxSelectedTools: 2 }, + ); + + expect(requests).toHaveLength(2); + expect(requests[0]).toEqual({ + pass: "initial", + query: "Arrange the appointment, then tell the teammate.", + tool_hints: [], + }); + expect(requests[1].pass).toBe("refined"); + expect(requests[1].tool_hints).toContain("calendar_find"); + expect(requests[1].tool_hints).toContain("email_send"); + expect(result.disposition).toBe("routed"); + expect(result.selected_tool_names).toEqual(["calendar_find", "email_send"]); + expect(result.evidence).toMatchObject({ + initial_subtask_count: 2, + refined_subtask_count: 2, + decomposition_passes: 2, + selected_tool_count: 2, + selected_tool_names: ["calendar_find", "email_send"], + fallback: null, + }); + expect(result.evidence.initial_candidate_counts).toEqual([4, 4]); + expect(result.evidence.final_candidate_counts).toEqual([4, 4]); + expect(result.evidence.final_candidate_tool_names[0][0]).toBe("calendar_find"); + expect(result.evidence.final_candidate_tool_names[1][0]).toBe("email_send"); + expect(JSON.stringify(result.evidence)).not.toContain("Arrange the appointment"); + expect(JSON.stringify(result.evidence)).not.toContain("locate the appointment"); + expect(embedder.calls).toHaveLength(3); + }); + + it("fails open for malformed decomposition and unavailable retrieval", async () => { + const catalog = [ + catalogEntry("calendar_find", "Find a calendar event."), + catalogEntry("email_send", "Send an email message."), + ]; + const malformed = await routeCompositionalTools("do work", catalog, { + decomposer: { + async decompose() { + return { steps: ["do work"] }; + }, + }, + embedder: new KeywordEmbedder(), + }); + expect(malformed.disposition).toBe("passthrough"); + expect(malformed.evidence.fallback).toBe("initial-decomposition-malformed"); + expect(selectRoutedToolDefinitions(catalog, malformed)).toEqual( + catalog.map((entry) => entry.definition), + ); + + const unavailable = await routeCompositionalTools("find the appointment", catalog, { + decomposer: { + async decompose() { + return ["find the calendar event"]; + }, + }, + embedder: { + async embed() { + throw new Error("embedding service unavailable"); + }, + }, + }); + expect(unavailable.disposition).toBe("passthrough"); + expect(unavailable.evidence.fallback).toBe("catalog-embedding-failed"); + expect(unavailable.selected_tool_names).toEqual([]); + + const disagreement = await routeCompositionalTools("find the appointment", catalog, { + decomposer: { + async decompose(request) { + return request.pass === "initial" ? ["find the calendar event"] : []; + }, + }, + embedder: new KeywordEmbedder(), + }); + expect(disagreement.disposition).toBe("passthrough"); + expect(disagreement.evidence.fallback).toBe("refinement-no-tool-disagreement"); + }); + + it("treats an empty initial decomposition as a valid no-tool route", async () => { + const embed = vi.fn(); + const decompose = vi.fn().mockResolvedValue([]); + const catalog = [catalogEntry("calendar_find", "Find a calendar event.")]; + + const result = await routeCompositionalTools("Answer without tools.", catalog, { + decomposer: { decompose }, + embedder: { embed }, + }); + + expect(result.disposition).toBe("routed"); + expect(result.selected_tool_names).toEqual([]); + expect(result.evidence).toMatchObject({ + initial_subtask_count: 0, + refined_subtask_count: 0, + decomposition_passes: 1, + hint_count: 0, + selected_tool_count: 0, + fallback: null, + }); + expect(decompose).toHaveBeenCalledOnce(); + expect(embed).not.toHaveBeenCalled(); + expect(selectRoutedToolDefinitions(catalog, result)).toEqual([]); + }); + + it("returns original schema objects in selected order and fails open above the cap", async () => { + const calendar = catalogEntry("calendar_find", "Find a calendar event."); + const email = catalogEntry("email_send", "Send an email message."); + const catalog = [email, calendar]; + const originalSchemas = structuredClone(catalog.map((entry) => entry.definition)); + const decomposer: TaskDecomposer = { + async decompose(request) { + return request.pass === "initial" + ? ["calendar event", "email message"] + : ["find calendar event", "send email message"]; + }, + }; + const result = await routeCompositionalTools("schedule and notify", catalog, { + decomposer, + embedder: new KeywordEmbedder(), + maxSelectedTools: 2, + }); + const selected = selectRoutedToolDefinitions(catalog, result); + + expect(selected).toEqual([calendar.definition, email.definition]); + expect(selected[0]).toBe(calendar.definition); + expect(selected[1]).toBe(email.definition); + expect(catalog.map((entry) => entry.definition)).toEqual(originalSchemas); + + const capped = await routeCompositionalTools("schedule and notify", catalog, { + decomposer, + embedder: new KeywordEmbedder(), + maxSelectedTools: 1, + }); + expect(capped.disposition).toBe("passthrough"); + expect(capped.evidence.fallback).toBe("selection-limit-exceeded"); + const passthrough = selectRoutedToolDefinitions(catalog, capped); + expect(passthrough).toEqual([email.definition, calendar.definition]); + expect(passthrough[0]).toBe(email.definition); + expect(passthrough[1]).toBe(calendar.definition); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-routing-acceptance.test.ts b/test/performance/tool-disclosure-compositional-routing-acceptance.test.ts new file mode 100644 index 0000000000..a3aeba26ff --- /dev/null +++ b/test/performance/tool-disclosure-compositional-routing-acceptance.test.ts @@ -0,0 +1,353 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + routeCompositionalTools, + routeCompositionalToolsPaired, +} from "../../scripts/performance/tool-disclosure/compositional-tool-router"; +import { + COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + type CompositionalRoutingCasePrediction, + type CompositionalRoutingVariant, + type CompositionalRoutingVariantInput, + compareCompositionalRoutingVariants, + DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS, + evaluateCompositionalRoutingGates, + evaluateCompositionalRoutingVariant, +} from "../../scripts/performance/tool-disclosure/compositional-tool-routing-acceptance"; +import { PortableHashingTextEmbedder } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-adapters"; + +const toolNames = COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.map((tool) => tool.name); + +function perfectPrediction(variant: CompositionalRoutingVariant): CompositionalRoutingVariantInput { + return { + variant, + cases: COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.map((fixture) => { + const expectedNames = fixture.expected_steps.map((expected) => expected.tool_name); + return { + case_id: fixture.id, + disposition: "routed" as const, + fallback: null, + steps: fixture.expected_steps.map((expected, index) => ({ + subtask: `${fixture.id} scripted atomic step ${index + 1}`, + ranked_tool_names: [ + expected.tool_name, + ...toolNames.filter((name) => name !== expected.tool_name), + ], + })), + selected_tool_names: expectedNames, + }; + }), + }; +} + +function replaceCase( + input: CompositionalRoutingVariantInput, + caseId: string, + update: (prediction: CompositionalRoutingCasePrediction) => CompositionalRoutingCasePrediction, +): CompositionalRoutingVariantInput { + return { + ...input, + cases: input.cases.map((prediction) => + prediction.case_id === caseId ? update(prediction) : prediction, + ), + }; +} + +describe("compositional routing acceptance corpus", () => { + it("contains distinct metadata and unambiguous cases from zero through five steps", () => { + expect(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS).toHaveLength(20); + expect(new Set(toolNames).size).toBe(toolNames.length); + expect( + new Set(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.map((tool) => tool.description)).size, + ).toBe(COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS.length); + + const stepCounts = COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.map( + (fixture) => fixture.expected_steps.length, + ); + expect(new Set(stepCounts)).toEqual(new Set([0, 1, 2, 3, 4, 5])); + expect(stepCounts.reduce((total, count) => total + count, 0)).toBe(18); + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + expect(fixture.prompt.trim()).not.toBe(""); + for (const expected of fixture.expected_steps) { + expect(toolNames).toContain(expected.tool_name); + expect(fixture.prompt.toLocaleLowerCase("en-US")).toContain(expected.prompt_cue); + expect(fixture.prompt).not.toContain(expected.tool_name); + } + } + }); + + it("scores perfect scripted decomposition, retrieval, selection, and no-tool behavior", () => { + const metrics = evaluateCompositionalRoutingVariant(perfectPrediction("refined")); + + expect(metrics).toMatchObject({ + variant: "refined", + case_count: 8, + expected_step_count: 18, + predicted_step_count: 18, + decomposition_exact_rate: 1, + decomposition_within_one_rate: 1, + exact_tool_recall_at_1: 1, + exact_tool_recall_at_10: 1, + chain_exact_selection_rate: 1, + selection_count_exact_rate: 1, + selected_tool_count: 18, + mean_selected_tool_count: 2.25, + max_selected_tool_count: 5, + no_tool_case_count: 1, + no_tool_exact_rate: 1, + }); + expect(metrics.cases.flatMap((fixture) => fixture.expected_tool_ranks)).toEqual( + Array.from({ length: 18 }, () => 1), + ); + expect(evaluateCompositionalRoutingGates(metrics)).toMatchObject({ passed: true, reasons: [] }); + }); + + it("passes the acceptance gates through the real router when decomposition is held fixed", async () => { + const embedder = new PortableHashingTextEmbedder(); + const predictions: CompositionalRoutingCasePrediction[] = []; + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + const subtasks = fixture.expected_steps.map((expected) => expected.capability); + const routed = await routeCompositionalTools( + fixture.prompt, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + { + embedder, + decomposer: { decompose: async () => subtasks }, + }, + ); + expect(routed.disposition, fixture.id).toBe("routed"); + predictions.push({ + case_id: fixture.id, + disposition: routed.disposition, + fallback: routed.evidence.fallback, + steps: routed.evidence.final_candidate_tool_names.map((ranked, index) => ({ + subtask: subtasks[index] ?? "no external capability", + ranked_tool_names: ranked, + })), + selected_tool_names: routed.selected_tool_names, + }); + } + + const metrics = evaluateCompositionalRoutingVariant({ variant: "refined", cases: predictions }); + expect(metrics).toMatchObject({ + decomposition_exact_rate: 1, + exact_tool_recall_at_1: 1, + exact_tool_recall_at_10: 1, + chain_exact_selection_rate: 1, + selection_count_exact_rate: 1, + max_selected_tool_count: 5, + no_tool_exact_rate: 1, + }); + expect(evaluateCompositionalRoutingGates(metrics)).toMatchObject({ passed: true, reasons: [] }); + }); + + it("shares the initial model output in the scripted initial-versus-refined control", async () => { + const embedder = new PortableHashingTextEmbedder(); + const initialCases: CompositionalRoutingCasePrediction[] = []; + const refinedCases: CompositionalRoutingCasePrediction[] = []; + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + const refinedSubtasks = fixture.expected_steps.map((expected) => expected.capability); + const initialSubtasks = + refinedSubtasks.length <= 1 + ? refinedSubtasks + : [`complete the ${refinedSubtasks.length} requested operations`]; + const pair = await routeCompositionalToolsPaired( + fixture.prompt, + COMPOSITIONAL_ROUTING_ACCEPTANCE_TOOLS, + { + embedder, + decomposer: { + decompose: async (request) => + request.pass === "refined" ? refinedSubtasks : initialSubtasks, + }, + }, + ); + expect(pair.shared_initial_decomposition, fixture.id).toBe(true); + initialCases.push({ + case_id: fixture.id, + disposition: pair.initial.disposition, + fallback: pair.initial.evidence.fallback, + steps: pair.initial.evidence.final_candidate_tool_names.map((ranked, index) => ({ + subtask: initialSubtasks[index] ?? "no external capability", + ranked_tool_names: ranked, + })), + selected_tool_names: pair.initial.selected_tool_names, + }); + refinedCases.push({ + case_id: fixture.id, + disposition: pair.refined.disposition, + fallback: pair.refined.evidence.fallback, + steps: pair.refined.evidence.final_candidate_tool_names.map((ranked, index) => ({ + subtask: refinedSubtasks[index] ?? "no external capability", + ranked_tool_names: ranked, + })), + selected_tool_names: pair.refined.selected_tool_names, + }); + } + + const comparison = compareCompositionalRoutingVariants( + { variant: "initial", cases: initialCases }, + { variant: "refined", cases: refinedCases }, + ); + + expect(comparison.initial.decomposition_exact_rate).toBeLessThan(1); + expect(comparison.refined.decomposition_exact_rate).toBe(1); + expect(comparison.refined.exact_tool_recall_at_1).toBe(1); + expect(comparison.refined_minus_initial.decomposition_exact_rate).toBeGreaterThan(0); + expect(comparison.refined_not_worse).toBe(true); + expect(comparison.refined_gate.passed).toBe(true); + expect(comparison.passed).toBe(true); + }); + + it("compares scripted initial and refined inputs without treating them as empirical evidence", () => { + let initial = perfectPrediction("initial"); + initial = replaceCase(initial, "route-chain-three-analysis-01", (prediction) => ({ + ...prediction, + steps: prediction.steps.slice(0, 2), + selected_tool_names: prediction.selected_tool_names.slice(0, 2), + })); + initial = replaceCase(initial, "route-single-bars-01", (prediction) => ({ + ...prediction, + steps: prediction.steps.map((step) => ({ + ...step, + ranked_tool_names: [ + "route_render_line_chart", + ...step.ranked_tool_names.filter((name) => name !== "route_render_line_chart"), + ], + })), + selected_tool_names: ["route_render_line_chart"], + })); + + const comparison = compareCompositionalRoutingVariants(initial, perfectPrediction("refined")); + + expect(comparison.initial.decomposition_exact_rate).toBeLessThan(1); + expect(comparison.initial.exact_tool_recall_at_1).toBeLessThan(1); + expect(comparison.initial.chain_exact_selection_rate).toBeLessThan(1); + expect(comparison.refined_minus_initial.decomposition_exact_rate).toBeGreaterThan(0); + expect(comparison.refined_minus_initial.exact_tool_recall_at_1).toBeGreaterThan(0); + expect(comparison.refined_not_worse).toBe(true); + expect(comparison.refined_gate.passed).toBe(true); + expect(comparison.passed).toBe(true); + }); + + it("fails strict gates when one selected chain is wrong", () => { + const refined = replaceCase( + perfectPrediction("refined"), + "route-single-bars-01", + (prediction) => ({ + ...prediction, + selected_tool_names: ["route_render_line_chart"], + }), + ); + const metrics = evaluateCompositionalRoutingVariant(refined); + const gate = evaluateCompositionalRoutingGates(metrics); + + expect(metrics.exact_tool_recall_at_1).toBe(1); + expect(metrics.chain_exact_selection_rate).toBe(7 / 8); + expect(gate.passed).toBe(false); + expect(gate.reasons).toContain("chain_exact_selection_rate 0.875 is below 1"); + }); + + it("counts a no-tool decomposition or selection as a control failure", () => { + const refined = replaceCase(perfectPrediction("refined"), "route-no-tool-01", (prediction) => ({ + ...prediction, + steps: [ + { + subtask: "unnecessary external lookup", + ranked_tool_names: ["route_lookup_directory_contact"], + }, + ], + selected_tool_names: ["route_lookup_directory_contact"], + })); + const metrics = evaluateCompositionalRoutingVariant(refined); + + expect(metrics.no_tool_exact_rate).toBe(0); + expect(metrics.selection_count_exact_rate).toBe(7 / 8); + expect(metrics.max_selected_tool_count).toBe(5); + expect(evaluateCompositionalRoutingGates(metrics).passed).toBe(false); + }); + + it("cannot score a failed-open no-tool route as successful", () => { + const refined = replaceCase(perfectPrediction("refined"), "route-no-tool-01", (prediction) => ({ + ...prediction, + disposition: "passthrough", + fallback: "initial-decomposition-failed", + selected_tool_names: [], + })); + const metrics = evaluateCompositionalRoutingVariant(refined); + + expect(metrics.routing_success_rate).toBe(7 / 8); + expect(metrics.route_failure_case_count).toBe(1); + expect(metrics.no_tool_exact_rate).toBe(0); + expect(metrics.chain_exact_forwarding_rate).toBe(7 / 8); + expect(metrics.forwarding_count_exact_rate).toBe(7 / 8); + expect(metrics.max_forwarded_tool_count).toBe(20); + expect(metrics.cases[0]).toMatchObject({ + disposition: "passthrough", + routing_succeeded: false, + forwarded_tool_count: 20, + no_tool_exact: false, + }); + const permissive = Object.fromEntries( + Object.entries(DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS).map(([key, value]) => [ + key, + key.startsWith("min_") ? 0 : Math.max(value, 20), + ]), + ) as unknown as typeof DEFAULT_COMPOSITIONAL_ROUTING_GATE_THRESHOLDS; + expect(evaluateCompositionalRoutingGates(metrics, permissive)).toMatchObject({ + passed: false, + reasons: ["route_failure_case_count 1 must be zero"], + }); + }); + + it("rejects incomplete, duplicate, and unknown-tool evidence", () => { + const perfect = perfectPrediction("refined"); + expect(() => + evaluateCompositionalRoutingVariant({ ...perfect, cases: perfect.cases.slice(1) }), + ).toThrow(/exactly 8 cases/); + expect(() => + evaluateCompositionalRoutingVariant({ + ...perfect, + cases: [perfect.cases[0], ...perfect.cases.slice(0, -1)], + }), + ).toThrow(/duplicate case/); + expect(() => + evaluateCompositionalRoutingVariant( + replaceCase(perfect, "route-single-csv-01", (prediction) => ({ + ...prediction, + selected_tool_names: ["route_unknown"], + })), + ), + ).toThrow(/selected unknown tool/); + expect(() => + evaluateCompositionalRoutingVariant( + replaceCase(perfect, "route-single-csv-01", (prediction) => ({ + ...prediction, + steps: prediction.steps.map((step) => ({ + ...step, + ranked_tool_names: [step.ranked_tool_names[0], step.ranked_tool_names[0]], + })), + })), + ), + ).toThrow(/ranking contains duplicates/); + expect(() => + evaluateCompositionalRoutingVariant( + replaceCase(perfect, "route-single-csv-01", (prediction) => ({ + ...prediction, + fallback: "initial-decomposition-failed", + })), + ), + ).toThrow(/inconsistent disposition and fallback/); + expect(() => + evaluateCompositionalRoutingVariant( + replaceCase(perfect, "route-single-csv-01", (prediction) => ({ + ...prediction, + disposition: "passthrough", + })), + ), + ).toThrow(/inconsistent disposition and fallback/); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-routing-adapters.test.ts b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts new file mode 100644 index 0000000000..7b821250df --- /dev/null +++ b/test/performance/tool-disclosure-compositional-routing-adapters.test.ts @@ -0,0 +1,384 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + createOpenAIChatTaskDecomposer, + createOpenAITextEmbedder, + MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS, + MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS, + MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS, + type ModelUsageEvent, + PortableHashingTextEmbedder, +} from "../../scripts/performance/tool-disclosure/compositional-tool-routing-adapters"; + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("independent compositional tool model adapters", () => { + it("requests both decomposition passes without copying request text into usage evidence", async () => { + const responses = [["collect the records"], ["fetch records", "send a notification"]]; + const calls: Array<{ url: string; headers: Headers; body: Record }> = []; + const fetchMock = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { + calls.push({ + url: String(url), + headers: new Headers(init?.headers), + body: JSON.parse(String(init?.body)) as Record, + }); + const content = JSON.stringify({ subtasks: responses[calls.length - 1] }); + return new Response( + JSON.stringify({ + choices: [{ message: { content: `\`\`\`json\n${content}\n\`\`\`` } }], + usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test/v1", + model: "test-model", + allowRemote: true, + apiKey: "private-api-key", + reasoningControl: "enable_thinking_false", + jsonObjectResponse: true, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ + pass: "initial", + query: "private request text", + tool_hints: [], + }), + ).resolves.toEqual(responses[0]); + await expect( + decomposer.decompose({ + pass: "refined", + query: "private request text", + tool_hints: ["route_fetch", "route_notify"], + }), + ).resolves.toEqual(responses[1]); + + expect(calls).toHaveLength(2); + expect(calls[0].url).toBe("https://models.example.test/v1/chat/completions"); + expect(calls[0].headers.get("authorization")).toBe("Bearer private-api-key"); + expect(calls[0].body).toMatchObject({ + model: "test-model", + temperature: 0, + max_tokens: 256, + stream: false, + chat_template_kwargs: { enable_thinking: false }, + response_format: { type: "json_object" }, + }); + expect(JSON.stringify(calls[1].body)).toContain("route_fetch"); + expect(usage).toHaveLength(2); + expect(usage.map((event) => event.pass)).toEqual(["initial", "refined"]); + expect(usage[0]).toMatchObject({ + operation: "decomposition", + prompt_tokens: 12, + completion_tokens: 4, + total_tokens: 16, + }); + expect(JSON.stringify(usage)).not.toMatch(/private request|private-api-key|test-model/); + }); + + it("returns a fixed decomposition failure without relaying provider content", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("provider-private-error", { status: 500 })), + ); + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).rejects.toThrow("decomposition request failed"); + }); + + it("retries decomposition within the configured bound and records every attempt", async () => { + let attempts = 0; + vi.stubGlobal( + "fetch", + vi.fn(async () => { + attempts += 1; + return attempts === 1 + ? new Response("temporary failure", { status: 503 }) + : new Response( + JSON.stringify({ + choices: [{ message: { content: '{"subtasks":["retry succeeded"]}' } }], + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }), + ); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 2, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).resolves.toEqual(["retry succeeded"]); + expect(attempts).toBe(2); + expect(usage).toMatchObject([ + { attempt: 1, outcome: "failed" }, + { attempt: 2, outcome: "completed" }, + ]); + }); + + it("retries a schema-invalid success response and aggregates usage from both attempts", async () => { + const responses = [ + { + content: '{"subtasks":"not-an-array"}', + usage: { prompt_tokens: 7, completion_tokens: 3, total_tokens: 10 }, + }, + { + content: '{"subtasks":["retry succeeded"]}', + usage: { prompt_tokens: 8, completion_tokens: 4, total_tokens: 12 }, + }, + ]; + const fetchMock = vi.fn(async () => { + const response = responses[fetchMock.mock.calls.length - 1]; + return new Response( + JSON.stringify({ + choices: [{ message: { content: response.content } }], + usage: response.usage, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 2, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).resolves.toEqual(["retry succeeded"]); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(usage).toMatchObject([ + { + attempt: 1, + outcome: "failed", + prompt_tokens: 7, + completion_tokens: 3, + total_tokens: 10, + }, + { + attempt: 2, + outcome: "completed", + prompt_tokens: 8, + completion_tokens: 4, + total_tokens: 12, + }, + ]); + expect( + usage.reduce( + (totals, event) => ({ + prompt_tokens: totals.prompt_tokens + (event.prompt_tokens ?? 0), + completion_tokens: totals.completion_tokens + (event.completion_tokens ?? 0), + total_tokens: totals.total_tokens + (event.total_tokens ?? 0), + }), + { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + ), + ).toEqual({ prompt_tokens: 15, completion_tokens: 7, total_tokens: 22 }); + }); + + it("stops after the exact configured retry count is exhausted", async () => { + const fetchMock = vi.fn(async () => new Response("temporary failure", { status: 503 })); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 3, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).rejects.toThrow("decomposition request failed"); + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(usage).toMatchObject([ + { attempt: 1, outcome: "failed" }, + { attempt: 2, outcome: "failed" }, + { attempt: 3, outcome: "failed" }, + ]); + }); + + it("does not retry a non-retryable authentication response", async () => { + const fetchMock = vi.fn(async () => new Response("unauthorized", { status: 401 })); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 3, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ pass: "initial", query: "request", tool_hints: [] }), + ).rejects.toThrow("decomposition request failed"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(usage).toMatchObject([{ attempt: 1, outcome: "failed" }]); + }); + + it("does not retry after its parent signal aborts", async () => { + const controller = new AbortController(); + const fetchMock = vi.fn(async () => { + controller.abort(); + throw new DOMException("aborted", "AbortError"); + }); + vi.stubGlobal("fetch", fetchMock); + const usage: ModelUsageEvent[] = []; + const decomposer = createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + maxAttempts: 3, + onUsage: (event) => usage.push(event), + }); + + await expect( + decomposer.decompose({ + pass: "initial", + query: "request", + tool_hints: [], + signal: controller.signal, + }), + ).rejects.toThrow("decomposition request failed"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(usage).toMatchObject([{ attempt: 1, outcome: "failed" }]); + }); + + it("enforces attempt, request-timeout, and total decomposition budget ceilings", () => { + const baseOptions = { + baseUrl: "https://models.example.test", + model: "test-model", + allowRemote: true, + } as const; + + expect(() => + createOpenAIChatTaskDecomposer({ + ...baseOptions, + maxAttempts: MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS + 1, + }), + ).toThrow(`maxAttempts must not exceed ${MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS}`); + expect(() => + createOpenAIChatTaskDecomposer({ + ...baseOptions, + timeoutMs: MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS + 1, + }), + ).toThrow(`timeoutMs must not exceed ${MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS}`); + expect(() => + createOpenAITextEmbedder({ + ...baseOptions, + timeoutMs: MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS + 1, + }), + ).toThrow(`timeoutMs must not exceed ${MAX_COMPOSITIONAL_MODEL_TIMEOUT_MS}`); + expect(() => + createOpenAIChatTaskDecomposer({ + ...baseOptions, + maxAttempts: MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS, + timeoutMs: + Math.floor( + MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS / MAX_COMPOSITIONAL_DECOMPOSER_ATTEMPTS, + ) + 1, + }), + ).toThrow( + `timeoutMs * maxAttempts must not exceed ${MAX_COMPOSITIONAL_DECOMPOSITION_BUDGET_MS}`, + ); + }); + + it("batches embeddings and restores provider results to input order", async () => { + const observedInputs: string[][] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const payload = JSON.parse(String(init?.body)) as { input: string[] }; + observedInputs.push(payload.input); + const data = payload.input + .map((text, index) => ({ index, embedding: [text.length, index + 1] })) + .reverse(); + return new Response(JSON.stringify({ data, usage: { total_tokens: 3 } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }), + ); + const usage: ModelUsageEvent[] = []; + const embedder = createOpenAITextEmbedder({ + baseUrl: "https://models.example.test/v1", + model: "test-embedding-model", + allowRemote: true, + batchSize: 2, + onUsage: (event) => usage.push(event), + }); + + await expect(embedder.embed(["a", "bb", "ccc"])).resolves.toEqual([ + [1, 1], + [2, 2], + [3, 1], + ]); + expect(observedInputs).toEqual([["a", "bb"], ["ccc"]]); + expect(usage).toHaveLength(2); + expect(usage.every((event) => event.operation === "embedding")).toBe(true); + }); + + it("provides a deterministic dependency-free lexical adapter for smoke tests", async () => { + const embedder = new PortableHashingTextEmbedder(512); + const [first, second, different] = await embedder.embed([ + "send an email notification", + "email notification send", + "decompress a gzip archive", + ]); + const dot = (left: readonly number[], right: readonly number[]) => + left.reduce((sum, value, index) => sum + value * right[index], 0); + + expect(first).toEqual(await embedder.embed(["send an email notification"]).then(([v]) => v)); + expect(dot(first, second)).toBeGreaterThan(dot(first, different)); + }); + + it("requires HTTPS and explicit opt-in before sending content off-host", () => { + expect(() => + createOpenAIChatTaskDecomposer({ + baseUrl: "https://models.example.test/v1", + model: "test-model", + }), + ).toThrow("require allowRemote"); + expect(() => + createOpenAITextEmbedder({ + baseUrl: "http://models.example.test/v1", + model: "test-model", + allowRemote: true, + }), + ).toThrow("must use HTTPS"); + expect(() => + createOpenAIChatTaskDecomposer({ + baseUrl: "http://127.0.0.1:8000/v1", + model: "test-model", + reasoningControl: "invalid" as "thinking_false", + }), + ).toThrow("reasoningControl is not supported"); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-routing-run.test.ts b/test/performance/tool-disclosure-compositional-routing-run.test.ts new file mode 100644 index 0000000000..56281305db --- /dev/null +++ b/test/performance/tool-disclosure-compositional-routing-run.test.ts @@ -0,0 +1,271 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-acceptance"; +import { runCompositionalRoutingAcceptance } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-run"; + +afterEach(() => { + delete process.env.ROUTING_TEST_API_KEY; + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("compositional routing acceptance runner", () => { + it("runs a shared-initial paired evaluation and emits public-safe evidence", async () => { + const privateKey = "private-route-acceptance-key"; + process.env.ROUTING_TEST_API_KEY = privateKey; + const fetchMock = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const payload = JSON.parse(String(init?.body)) as { + messages: Array<{ role: string; content: string }>; + }; + const user = + [...payload.messages].reverse().find((message) => message.role === "user")?.content ?? ""; + const fixture = COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.find((candidate) => + user.includes(candidate.prompt), + ); + expect(fixture).toBeDefined(); + const matchedFixture = fixture as NonNullable; + return new Response( + JSON.stringify({ + choices: [ + { + message: { + content: JSON.stringify({ + subtasks: matchedFixture.expected_steps.map((expected) => expected.capability), + }), + }, + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + + const output = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + api_key_env: "ROUTING_TEST_API_KEY", + allow_remote: true, + reasoning_control: "enable_thinking_false", + json_object_response: true, + max_attempts: 2, + }, + embedding: { kind: "portable", dimensions: 1_024 }, + run_timeout_ms: 900_000, + }); + + expect(output).toMatchObject({ + schema_version: "nemoclaw.compositional_tool_routing_acceptance.v1", + claim_eligible: false, + configuration: { + decomposer_model: "test-decomposer", + decomposer_revision: "immutable-test-revision", + decomposer_reasoning_control: "enable_thinking_false", + decomposer_output_mode: "json-object", + decomposer_max_attempts: 2, + request_timeout_ms: 120_000, + run_timeout_ms: 900_000, + embedding_kind: "portable", + embedding_model: "portable-lexical-hashing", + embedding_revision: "builtin-v1", + top_k: 10, + hint_count: 12, + temperature: 0, + }, + corpus: { tool_count: 20, case_count: 8, expected_step_count: 18 }, + usage: { + decomposition: { + requests: 15, + failed_requests: 0, + prompt_tokens: 150, + completion_tokens: 30, + total_tokens: 180, + }, + embedding: { requests: 0, failed_requests: 0 }, + }, + execution: { + status: "completed", + completed_case_count: 8, + total_case_count: 8, + }, + acceptance_passed: true, + }); + expect(output.cases).toHaveLength(8); + expect(output.cases.every((entry) => entry.shared_initial_decomposition)).toBe(true); + expect(output.cases.every((entry) => entry.evaluation_status === "completed")).toBe(true); + const noTool = output.cases.find((entry) => entry.case_id === "route-no-tool-01"); + expect(noTool?.refined).toMatchObject({ + disposition: "routed", + forwarded_tool_names: [], + forwarded_tool_count: 0, + evidence: { fallback: null }, + }); + const toolCase = output.cases.find((entry) => entry.case_id === "route-single-csv-01"); + expect(toolCase?.refined.forwarded_tool_names).toEqual( + toolCase?.refined.evidence.selected_tool_names, + ); + expect(output.comparison.refined_gate.passed).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(15); + const serialized = JSON.stringify(output); + expect(serialized).not.toContain(privateKey); + expect(serialized).not.toContain("models.example.test"); + for (const fixture of COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES) { + expect(serialized).not.toContain(fixture.prompt); + } + }); + + it("fails acceptance and records full-catalog forwarding when no-tool routing fails open", async () => { + process.env.ROUTING_TEST_API_KEY = "private-route-acceptance-key"; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const payload = JSON.parse(String(init?.body)) as { + messages: Array<{ role: string; content: string }>; + }; + const user = + [...payload.messages].reverse().find((message) => message.role === "user")?.content ?? ""; + const fixture = COMPOSITIONAL_ROUTING_ACCEPTANCE_CASES.find((candidate) => + user.includes(candidate.prompt), + ); + expect(fixture).toBeDefined(); + const matchedFixture = fixture as NonNullable; + const content = + matchedFixture.id === "route-no-tool-01" + ? "not-json" + : JSON.stringify({ + subtasks: matchedFixture.expected_steps.map((expected) => expected.capability), + }); + return new Response(JSON.stringify({ choices: [{ message: { content } }] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }), + ); + + const output = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + api_key_env: "ROUTING_TEST_API_KEY", + allow_remote: true, + }, + embedding: { kind: "portable" }, + }); + + expect(output.acceptance_passed).toBe(false); + expect(output.comparison.refined.route_failure_case_count).toBe(1); + expect(output.comparison.refined_gate.reasons).toContain( + "route_failure_case_count 1 must be zero", + ); + const noTool = output.cases.find((entry) => entry.case_id === "route-no-tool-01"); + expect(noTool?.refined).toMatchObject({ + disposition: "passthrough", + forwarded_tool_count: 20, + evidence: { fallback: "initial-decomposition-failed" }, + }); + expect(noTool?.refined.forwarded_tool_names).toHaveLength(20); + }); + + it("rejects request and overall timeouts outside their supported bounds", async () => { + const baseConfig = { + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + allow_remote: true, + }, + embedding: { kind: "portable" as const }, + }; + + await expect( + runCompositionalRoutingAcceptance({ ...baseConfig, timeout_ms: 300_001 }), + ).rejects.toThrow("timeout_ms must be a positive safe integer no greater than 300000"); + await expect( + runCompositionalRoutingAcceptance({ ...baseConfig, run_timeout_ms: 2_700_001 }), + ).rejects.toThrow("run_timeout_ms must be a positive safe integer no greater than 2700000"); + }); + + it("uses the overall deadline to stop outstanding decomposition work", async () => { + const fetchMock = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + const signal = init?.signal as AbortSignal; + signal.throwIfAborted(); + return new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError")), { + once: true, + }); + }); + }); + vi.stubGlobal("fetch", fetchMock); + + const output = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + allow_remote: true, + max_attempts: 3, + }, + embedding: { kind: "portable" }, + run_timeout_ms: 10, + }); + + expect(output.acceptance_passed).toBe(false); + expect(output.configuration.run_timeout_ms).toBe(10); + expect(output.execution).toEqual({ + status: "timed-out", + completed_case_count: 0, + total_case_count: 8, + }); + expect(output.comparison.refined.route_failure_case_count).toBe(8); + expect(output.comparison.reasons).toContain("run deadline exceeded after 0 of 8 cases"); + expect( + output.cases.every( + (entry) => + entry.evaluation_status === "run-deadline-exceeded" && + entry.initial.evidence.fallback === "run-deadline-exceeded" && + entry.refined.evidence.fallback === "run-deadline-exceeded", + ), + ).toBe(true); + expect(output.usage.decomposition).toMatchObject({ requests: 1, failed_requests: 1 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("cannot pass after the monotonic run deadline when timer callbacks are delayed", async () => { + let monotonicNow = 0; + vi.spyOn(performance, "now").mockImplementation(() => { + monotonicNow += 2; + return monotonicNow; + }); + const fetchMock = vi.fn( + async () => + new Response(JSON.stringify({ choices: [{ message: { content: '{"subtasks":[]}' } }] }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + + const output = await runCompositionalRoutingAcceptance({ + decomposer: { + base_url: "https://models.example.test/v1", + model: "test-decomposer", + revision: "immutable-test-revision", + allow_remote: true, + }, + embedding: { kind: "portable" }, + run_timeout_ms: 1, + }); + + expect(output.acceptance_passed).toBe(false); + expect(output.execution.status).toBe("timed-out"); + expect(output.comparison.reasons).toContain("run deadline exceeded after 0 of 8 cases"); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/test/performance/tool-disclosure-compositional-routing-transform.test.ts b/test/performance/tool-disclosure-compositional-routing-transform.test.ts new file mode 100644 index 0000000000..93b9b7146b --- /dev/null +++ b/test/performance/tool-disclosure-compositional-routing-transform.test.ts @@ -0,0 +1,506 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import http from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, describe, expect, it } from "vitest"; +import type { + DecompositionRequest, + TaskDecomposer, + TextEmbedder, +} from "../../scripts/performance/tool-disclosure/compositional-tool-router"; +import { CompositionalToolRoutingTransform } from "../../scripts/performance/tool-disclosure/compositional-tool-routing-transform"; +import { createToolDisclosureRecordingProxy } from "../../scripts/performance/tool-disclosure/recorder"; + +function vectorFor(text: string): number[] { + const normalized = text.toLowerCase(); + return normalized.includes("calendar") + ? [1, 0, 0] + : normalized.includes("email") + ? [0, 1, 0] + : [0, 0, 1]; +} + +const embedder: TextEmbedder = { + embed: async (texts) => texts.map(vectorFor), +}; + +const cleanup: Array<() => Promise> = []; + +afterEach(async () => { + await Promise.allSettled(cleanup.splice(0).map((close) => close())); +}); + +function tools(): unknown[] { + return [ + { + type: "function", + function: { name: "core_shell", description: "Run a shell command.", parameters: {} }, + }, + { + type: "function", + function: { + name: "route_calendar_list", + description: "List calendar events.", + parameters: {}, + }, + }, + { + type: "function", + function: { + name: "route_storage_delete", + description: "Delete an object from storage.", + parameters: {}, + }, + }, + { + type: "function", + function: { + name: "route_email_send", + description: "Send an email message.", + parameters: {}, + }, + }, + ]; +} + +function requestBody(prompt = "List today's calendar events and send them by email."): Buffer { + return Buffer.from( + JSON.stringify({ + model: "private-model", + messages: [{ role: "user", content: prompt }], + tools: tools(), + }), + "utf8", + ); +} + +function input(body: Buffer, modelCallSequence = 1) { + return { + runId: "route-test-run", + endpoint: "chat-completions" as const, + method: "POST" as const, + modelCallSequence, + body, + signal: new AbortController().signal, + }; +} + +function names(body: Buffer): string[] { + const payload = JSON.parse(body.toString("utf8")) as { tools: unknown[] }; + return payload.tools.map((tool) => { + const value = tool as { function: { name: string } }; + return value.function.name; + }); +} + +describe("independent compositional tool request transform", () => { + it("filters the schemas measured and forwarded by the recording proxy", async () => { + let upstreamBody = ""; + const upstream = http.createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => { + upstreamBody = Buffer.concat(chunks).toString("utf8"); + response.end("{}"); + }); + }); + await new Promise((resolve, reject) => { + upstream.once("error", reject); + upstream.listen(0, "127.0.0.1", resolve); + }); + cleanup.push(() => new Promise((resolve) => upstream.close(() => resolve()))); + const address = upstream.address() as AddressInfo; + const transform = new CompositionalToolRoutingTransform({ + decomposer: { + decompose: async () => ["list calendar events", "send an email message"], + }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const proxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: `http://127.0.0.1:${address.port}`, + requestTransform: transform.requestTransform, + }); + const proxyAddress = await proxy.start(); + cleanup.push(() => proxy.stop()); + + proxy.beginRun("proxy-route-run"); + const response = await fetch(`${proxyAddress.base_url}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: requestBody(), + }); + await response.arrayBuffer(); + const [event] = proxy.endRun(); + + expect(response.status).toBe(200); + expect(names(Buffer.from(upstreamBody))).toEqual([ + "core_shell", + "route_calendar_list", + "route_email_send", + ]); + expect(event).toMatchObject({ + visible_tool_count: 3, + tool_names: ["core_shell", "route_calendar_list", "route_email_send"], + }); + expect(await transform.consumeEvidence("proxy-route-run")).toHaveLength(1); + }); + + it("runs two decomposition passes, filters only routable tools, and caches the route", async () => { + const requests: DecompositionRequest[] = []; + const decomposer: TaskDecomposer = { + decompose: async (request) => { + requests.push(request); + return ["list calendar events", "send an email message"]; + }, + }; + const transform = new CompositionalToolRoutingTransform({ + decomposer, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + + const body = requestBody(); + const first = await transform.requestTransform(input(body)); + const second = await transform.requestTransform(input(body, 2)); + + expect(names(first)).toEqual(["core_shell", "route_calendar_list", "route_email_send"]); + expect(second.equals(first)).toBe(true); + expect(requests).toHaveLength(2); + expect(requests.map((request) => request.pass)).toEqual(["initial", "refined"]); + expect(requests[1].tool_hints).toContain("route_calendar_list"); + expect(requests[1].tool_hints).toContain("route_email_send"); + + const [evidence] = await transform.consumeEvidence("route-test-run"); + expect(evidence).toMatchObject({ + source_tool_count: 4, + routable_tool_count: 3, + preserved_tool_count: 1, + forwarded_tool_count: 3, + cache_hits: 1, + index_cache_hit: false, + transform_bypass: null, + routing: { + initial_subtask_count: 2, + refined_subtask_count: 2, + decomposition_passes: 2, + selected_tool_count: 2, + selected_tool_names: ["route_calendar_list", "route_email_send"], + fallback: null, + }, + }); + expect(JSON.stringify(evidence)).not.toContain("today's calendar"); + expect(await transform.consumeEvidence("route-test-run")).toEqual([]); + }); + + it("keeps the full request byte-for-byte when routing falls back", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ({ malformed: true }) }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const body = requestBody("private fallback prompt"); + + const forwarded = await transform.requestTransform(input(body)); + + expect(forwarded.equals(body)).toBe(true); + const [evidence] = await transform.consumeEvidence("route-test-run"); + expect(evidence).toMatchObject({ + forwarded_tool_count: 4, + routing: { + decomposition_passes: 1, + fallback: "initial-decomposition-malformed", + }, + }); + expect(JSON.stringify(evidence)).not.toContain("private fallback prompt"); + }); + + it("rejects a routing fallback in strict mode while retaining its evidence", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ({ malformed: true }) }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + requireRouting: true, + }); + + await expect(transform.requestTransform(input(requestBody()))).rejects.toThrow( + "required compositional routing bypassed: routing fallback: initial-decomposition-malformed", + ); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { routing: { fallback: "initial-decomposition-malformed" } }, + ]); + }); + + it("treats an empty decomposition as an intentional no-tool route", async () => { + let embeddingCalls = 0; + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => [] }, + embedder: { + embed: async () => { + embeddingCalls += 1; + return []; + }, + }, + isRoutableTool: (name) => name.startsWith("route_"), + }); + + const forwarded = await transform.requestTransform( + input(requestBody("Reply with the literal phrase CONTROL and use no tools.")), + ); + + expect(names(forwarded)).toEqual(["core_shell"]); + expect(embeddingCalls).toBe(1); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { + forwarded_tool_count: 1, + routing: { + initial_subtask_count: 0, + refined_subtask_count: 0, + decomposition_passes: 1, + selected_tool_count: 0, + fallback: null, + }, + }, + ]); + }); + + it("leaves unsupported endpoints untouched and does not retain evidence", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ["unused"] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const body = requestBody(); + const forwarded = await transform.requestTransform({ + ...input(body), + endpoint: "responses", + }); + + expect(forwarded.equals(body)).toBe(true); + expect(await transform.consumeEvidence("route-test-run")).toEqual([]); + }); + + it("reuses the prepared catalog index across run IDs", async () => { + const embeddingBatchSizes: number[] = []; + const transform = new CompositionalToolRoutingTransform({ + decomposer: { + decompose: async () => ["list calendar events", "send an email message"], + }, + embedder: { + embed: async (texts) => { + embeddingBatchSizes.push(texts.length); + return texts.map(vectorFor); + }, + }, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const body = requestBody(); + + await transform.requestTransform(input(body)); + await transform.requestTransform({ ...input(body), runId: "route-test-run-2" }); + + const [first] = await transform.consumeEvidence("route-test-run"); + const [second] = await transform.consumeEvidence("route-test-run-2"); + expect(first.index_cache_hit).toBe(false); + expect(second.index_cache_hit).toBe(true); + expect(embeddingBatchSizes).toEqual([3, 2, 2, 2, 2]); + }); + + it("lets one request abort without cancelling shared route or index work", async () => { + let releaseInitial: ((subtasks: string[]) => void) | undefined; + let markInitialStarted: (() => void) | undefined; + const initialStarted = new Promise((resolve) => { + markInitialStarted = resolve; + }); + const decompositionSignals: Array = []; + const embeddingSignals: Array = []; + const transform = new CompositionalToolRoutingTransform({ + decomposer: { + decompose: async (request) => { + decompositionSignals.push(request.signal); + return request.pass === "refined" + ? ["list calendar events"] + : (markInitialStarted?.(), + new Promise((resolve) => { + releaseInitial = resolve; + })); + }, + }, + embedder: { + embed: async (texts, signal) => { + embeddingSignals.push(signal); + return texts.map(vectorFor); + }, + }, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const firstController = new AbortController(); + const secondController = new AbortController(); + const body = requestBody("List today's calendar events."); + const first = transform.requestTransform({ + ...input(body), + signal: firstController.signal, + }); + await initialStarted; + const second = transform.requestTransform({ + ...input(body, 2), + signal: secondController.signal, + }); + const firstRejection = expect(first).rejects.toMatchObject({ name: "AbortError" }); + + firstController.abort(); + await firstRejection; + releaseInitial?.(["list calendar events"]); + + expect(names(await second)).toEqual(["core_shell", "route_calendar_list"]); + const [routeSignal] = decompositionSignals; + expect(routeSignal).toBeDefined(); + expect(decompositionSignals.every((signal) => signal === routeSignal)).toBe(true); + expect(routeSignal?.aborted).toBe(false); + expect(embeddingSignals[0]).toBeUndefined(); + expect(embeddingSignals.slice(1).every((signal) => signal === routeSignal)).toBe(true); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { cache_hits: 1, routing: { fallback: null } }, + ]); + }); + + it("cancels unfinished route work after its last waiter aborts and starts fresh work", async () => { + let initialCalls = 0; + let firstRouteSignal: AbortSignal | undefined; + let markFirstStarted: (() => void) | undefined; + const firstStarted = new Promise((resolve) => { + markFirstStarted = resolve; + }); + const transform = new CompositionalToolRoutingTransform({ + decomposer: { + decompose: async (request) => { + initialCalls += request.pass === "initial" ? 1 : 0; + const holdFirst = request.pass === "initial" && initialCalls === 1; + return holdFirst + ? ((firstRouteSignal = request.signal), + markFirstStarted?.(), + new Promise((_resolve, reject) => { + request.signal?.addEventListener( + "abort", + () => reject(new DOMException("routing aborted", "AbortError")), + { once: true }, + ); + })) + : ["list calendar events"]; + }, + }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const body = requestBody("List today's calendar events."); + const controller = new AbortController(); + const first = transform.requestTransform({ + ...input(body), + signal: controller.signal, + }); + await firstStarted; + const firstRejection = expect(first).rejects.toMatchObject({ name: "AbortError" }); + + controller.abort(); + await firstRejection; + + expect(firstRouteSignal?.aborted).toBe(true); + expect(await transform.consumeEvidence("route-test-run")).toEqual([]); + + const fresh = await transform.requestTransform(input(body, 2)); + expect(names(fresh)).toEqual(["core_shell", "route_calendar_list"]); + expect(initialCalls).toBe(2); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { cache_hits: 0, routing: { fallback: null } }, + ]); + }); + + it("preserves a request whose named tool choice conflicts with the route", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ["list calendar events"] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const payload = JSON.parse(requestBody().toString("utf8")) as Record; + payload.tool_choice = { + type: "function", + function: { name: "route_storage_delete" }, + }; + const body = Buffer.from(JSON.stringify(payload), "utf8"); + + const forwarded = await transform.requestTransform(input(body)); + + expect(forwarded.equals(body)).toBe(true); + const [evidence] = await transform.consumeEvidence("route-test-run"); + expect(evidence).toMatchObject({ + forwarded_tool_count: 4, + transform_bypass: "tool-choice-conflict", + }); + }); + + it("rejects a named tool-choice conflict in strict mode", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => ["list calendar events"] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + requireRouting: true, + }); + const payload = JSON.parse(requestBody().toString("utf8")) as Record; + payload.tool_choice = { + type: "function", + function: { name: "route_storage_delete" }, + }; + + await expect( + transform.requestTransform(input(Buffer.from(JSON.stringify(payload), "utf8"))), + ).rejects.toThrow( + "required compositional routing bypassed: named tool choice conflicts with routed catalog", + ); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { transform_bypass: "tool-choice-conflict", forwarded_tool_count: 4 }, + ]); + }); + + it("preserves a required-tool request when routing would expose no tools", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => [] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + }); + const payload = JSON.parse(requestBody().toString("utf8")) as { tools: unknown[] } & Record< + string, + unknown + >; + payload.tools = payload.tools.slice(1); + payload.tool_choice = "required"; + const body = Buffer.from(JSON.stringify(payload), "utf8"); + + expect((await transform.requestTransform(input(body))).equals(body)).toBe(true); + expect(await transform.consumeEvidence("route-test-run")).toMatchObject([ + { transform_bypass: "tool-choice-conflict", forwarded_tool_count: 3 }, + ]); + }); + + it("rejects an empty required-tool route in strict mode", async () => { + const transform = new CompositionalToolRoutingTransform({ + decomposer: { decompose: async () => [] }, + embedder, + isRoutableTool: (name) => name.startsWith("route_"), + requireRouting: true, + }); + const payload = JSON.parse(requestBody().toString("utf8")) as { tools: unknown[] } & Record< + string, + unknown + >; + payload.tools = payload.tools.slice(1); + payload.tool_choice = "required"; + + await expect( + transform.requestTransform(input(Buffer.from(JSON.stringify(payload), "utf8"))), + ).rejects.toThrow( + "required compositional routing bypassed: required tool choice has no routed tools", + ); + }); +}); diff --git a/test/performance/tool-disclosure-drivers.test.ts b/test/performance/tool-disclosure-drivers.test.ts new file mode 100644 index 0000000000..dca380c5bd --- /dev/null +++ b/test/performance/tool-disclosure-drivers.test.ts @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; + +import { describe, expect, it } from "vitest"; + +import { + buildAgentDriverCommand, + extractFinalAssistantOutput, + outputContainsOracle, +} from "../../scripts/performance/tool-disclosure/drivers"; + +describe("tool-disclosure agent drivers", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("builds a bounded sandbox command for %s", (agent) => { + const prompt = "Use the weather converter,\nthen return PERFORMANCE_TEST_OK_123"; + const command = buildAgentDriverCommand({ + agent, + sandboxName: "performance-test-openclaw-progressive-512", + prompt, + sessionId: "performance-test-session-1", + }); + expect(command.command).toBe("openshell"); + expect(command.args.slice(0, 5)).toEqual([ + "sandbox", + "exec", + "-n", + "performance-test-openclaw-progressive-512", + "--", + ]); + expect(command.args.join(" ")).not.toContain(prompt); + expect(command.args.every((argument) => !/[\r\n]/u.test(argument))).toBe(true); + expect(command.redactions).toContain(prompt); + }); + + it("uses an exact oracle and rejects unsafe identifiers", () => { + expect(outputContainsOracle("result PERFORMANCE_TEST_OK_123", "PERFORMANCE_TEST_OK_123")).toBe( + true, + ); + expect(outputContainsOracle("result PERFORMANCE_TEST_OK_123", "PERFORMANCE_TEST_OK_456")).toBe( + false, + ); + expect(() => + buildAgentDriverCommand({ + agent: "openclaw", + sandboxName: "bad name", + prompt: "x", + sessionId: "one", + }), + ).toThrow("sandboxName contains unsupported characters"); + }); + + it.skipIf(process.platform === "win32")("emits POSIX-valid driver scripts", () => { + for (const agent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { + const command = buildAgentDriverCommand({ + agent, + sandboxName: "performance-test-syntax", + prompt: "Return PERFORMANCE_TEST_OK_123", + sessionId: "performance-test-session-syntax", + }); + const syntax = spawnSync("sh", ["-n"], { + encoding: "utf8", + input: command.args.at(-1), + }); + expect(syntax.status, syntax.stderr).toBe(0); + } + }); + + it("streams the decoded Hermes payload directly to curl", () => { + const command = buildAgentDriverCommand({ + agent: "hermes", + sandboxName: "performance-test-hermes-stream", + prompt: "Return PERFORMANCE_TEST_OK_123", + sessionId: "performance-test-session-hermes-stream", + }); + const script = command.args.at(-1) ?? ""; + expect(script).toContain("base64 -d | curl"); + expect(script).toContain("--data-binary @-"); + expect(script).not.toContain("payload=$(printf"); + }); + + it("extracts assistant replies without accepting tool-result fields", () => { + expect( + extractFinalAssistantOutput( + "openclaw", + JSON.stringify({ + result: { payloads: [{ text: "FINAL_NONCE" }] }, + tool_result: { text: "TOOL_ONLY_NONCE" }, + }), + ), + ).toBe("FINAL_NONCE"); + expect( + extractFinalAssistantOutput( + "hermes", + JSON.stringify({ + choices: [{ message: { role: "assistant", content: "FINAL_NONCE" } }], + tool_result: "TOOL_ONLY_NONCE", + }), + ), + ).toBe("FINAL_NONCE"); + }); + + it("extracts the Deep Agents final block before trailing task status", () => { + const output = [ + "Running task non-interactively...", + "App: v0.1.30 | Agent: agent (default) | Model: example/model | Thread: thread-1", + "Starting LangGraph server...", + "✓ Loaded 1 MCP tool", + "✓ Server ready", + "🔧 Calling tool: search_tools (trace: TOOL_ONLY_NONCE)", + "🔧 Calling tool: performance_test_echo", + "Operation complete.", + "FINAL_NONCE", + "", + "✓ Task completed", + "", + "Agent active 1.3s", + ].join("\n"); + + expect(extractFinalAssistantOutput("langchain-deepagents-code", output)).toBe( + "Operation complete.\nFINAL_NONCE", + ); + expect( + extractFinalAssistantOutput( + "langchain-deepagents-code", + "✓ Server ready\n🔧 Calling tool: performance_test_echo (result: TOOL_ONLY_NONCE)\n✓ Task completed\nAgent active 1.3s", + ), + ).toBe(""); + }); +}); diff --git a/test/performance/tool-disclosure-execute.test.ts b/test/performance/tool-disclosure-execute.test.ts new file mode 100644 index 0000000000..04d55b0bca --- /dev/null +++ b/test/performance/tool-disclosure-execute.test.ts @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + type AttemptJournalEntry, + assertSetupRetryAllowed, + type CampaignAttestation, + classifyInvocationFailure, + materializeAttemptJournal, + nextSetupAttempt, + readAttestedVllmConfiguration, + recoverAttemptJournal, + selectInspectedContainer, +} from "../../scripts/performance/tool-disclosure/execute"; +import { validateCampaignAttestations } from "../../scripts/performance/tool-disclosure/run"; +import { + STATIC_CATALOG_SIZES, + TOOL_DISCLOSURE_AGENTS, + TOOL_DISCLOSURE_MODES, +} from "../../scripts/performance/tool-disclosure/schedule"; +import type { ToolDisclosureManifest } from "../../scripts/performance/tool-disclosure/types"; + +const hex = (value: number): string => value.toString(16).padStart(64, "0"); + +function attestationFixture(): { + manifest: ToolDisclosureManifest; + attestations: CampaignAttestation[]; +} { + const cells = TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => `${agent}:${mode}:${size}`), + ), + ); + const sandboxDigest = `sha256:${"a".repeat(64)}`; + const inferenceDigest = `sha256:${"b".repeat(64)}`; + const manifest = { + inference: { container_digest: inferenceDigest }, + environment: { + sandbox_image_digests: Object.fromEntries(cells.map((cell) => [cell, sandboxDigest])), + }, + } as ToolDisclosureManifest; + const attestations = [1, 2].map( + (campaign): CampaignAttestation => ({ + campaign_id: `campaign-${campaign}`, + vllm_process_start_time_seconds: campaign, + inference_container_id_sha256: hex(campaign), + inference_config_sha256: hex(100), + inference_image_digest: inferenceDigest, + sandbox_cells: cells.map((cell, index) => ({ + cell, + instance_id_sha256: hex(campaign * 100 + index), + status_sha256: hex(campaign * 1_000 + index), + image_digest: sandboxDigest, + })), + }), + ); + return { manifest, attestations }; +} + +describe("tool-disclosure attempt journal", () => { + it("gives timeout precedence over context-overflow text", () => { + expect( + classifyInvocationFailure({ + phase: "primary", + exitCode: null, + timedOut: true, + stdout: "", + stderr: "maximum context length exceeded", + }), + ).toBeUndefined(); + expect( + classifyInvocationFailure({ + phase: "primary", + exitCode: 1, + timedOut: false, + stdout: "", + stderr: "maximum context length exceeded", + }), + ).toBe("context-overflow"); + }); + + it("retries only failures that occur before agent invocation", () => { + expect(() => assertSetupRetryAllowed(false, "run-before-invocation")).not.toThrow(); + expect(() => + assertSetupRetryAllowed(true, "run-after-invocation", new Error("private detail")), + ).toThrow(/discard this campaign/u); + }); + + it("preserves exhausted setup attempts and refuses another resume", () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); + try { + const entry = { + raw: { run_id: "run-exhausted", failure_outcome: "setup-error" }, + run: { run_id: "run-exhausted", outcome: "setup-error" }, + } as unknown as AttemptJournalEntry; + const attempts = [entry, structuredClone(entry)]; + const serialized = `${attempts.map((attempt) => JSON.stringify(attempt)).join("\n")}\n`; + const file = path.join(output, "attempt-journal.jsonl"); + fs.writeFileSync(file, serialized); + + expect(nextSetupAttempt([entry], "run-exhausted", 1)).toBe(1); + const recovered = recoverAttemptJournal(output); + expect(recovered).toEqual(attempts); + expect(() => nextSetupAttempt(recovered, "run-exhausted", 1)).toThrow( + /exhausted setup retries/u, + ); + expect(fs.readFileSync(file, "utf8")).toBe(serialized); + } finally { + fs.rmSync(output, { recursive: true, force: true }); + } + }); + + it("rejects ambiguous Docker inspect output regardless of order", () => { + const expected = { Id: "expected", Image: "sha256:expected" }; + const other = { Id: "other", Image: "sha256:other" }; + expect(selectInspectedContainer([expected], expected.Id)).toEqual(expected); + expect(selectInspectedContainer([other, expected], expected.Id)).toBeUndefined(); + expect(selectInspectedContainer([expected, other], expected.Id)).toBeUndefined(); + }); + + it("attests vLLM arguments only from the intended process configuration", () => { + const expected: ToolDisclosureManifest["inference"] = { + api: "chat-completions", + model_id: "example/model", + model_revision: "revision-1", + container_image: "registry.example/vllm:1.0.0", + container_digest: `sha256:${"a".repeat(64)}`, + vllm_version: "1.0.0", + tool_call_parser: "tool-parser", + reasoning_parser: "reasoning-parser", + temperature: 0, + concurrency: 1, + prefix_caching_enabled: false, + public_vllm_flags: ["--enable-auto-tool-choice", "--max-model-len 8192"], + }; + const command = + "vllm serve example/model --revision revision-1 " + + "--tool-call-parser tool-parser --reasoning-parser reasoning-parser " + + "--enable-auto-tool-choice --max-model-len 8192"; + expect( + readAttestedVllmConfiguration( + { + Config: { + Entrypoint: ["/bin/bash"], + Cmd: ["-lc", command], + Env: ["VLLM_USE_V1=1", "UNRELATED=ignored"], + }, + }, + expected, + ), + ).toEqual({ + cmd: ["-lc", command], + entrypoint: ["/bin/bash"], + env: ["VLLM_USE_V1=1"], + }); + + const decoy = [ + expected.model_id, + expected.model_revision, + expected.tool_call_parser, + expected.reasoning_parser, + ...expected.public_vllm_flags, + ].join(" "); + expect( + readAttestedVllmConfiguration( + { + Config: { + Entrypoint: ["vllm"], + Cmd: ["serve", "different/model"], + Env: [`UNRELATED=${decoy}`], + Labels: { unrelated: decoy }, + }, + }, + expected, + ), + ).toBeUndefined(); + }); + + it("recovers one trailing partial append and deterministically materializes evidence", () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); + try { + const entry = { + raw: { run_id: "run-1", recorder_events: [], calls: [] }, + run: { run_id: "run-1", outcome: "setup-error" }, + } as unknown as AttemptJournalEntry; + fs.writeFileSync( + path.join(output, "attempt-journal.jsonl"), + `${JSON.stringify(entry)}\n{"raw":`, + ); + + const recovered = recoverAttemptJournal(output); + expect(recovered).toEqual([entry]); + expect(fs.readFileSync(path.join(output, "attempt-journal.jsonl"), "utf8")).toBe( + `${JSON.stringify(entry)}\n`, + ); + + materializeAttemptJournal(output, recovered); + expect(JSON.parse(fs.readFileSync(path.join(output, "raw-events.jsonl"), "utf8"))).toEqual( + entry.raw, + ); + expect(JSON.parse(fs.readFileSync(path.join(output, "runs.jsonl"), "utf8"))).toEqual( + entry.run, + ); + } finally { + fs.rmSync(output, { recursive: true, force: true }); + } + }); + + it("fails closed for a newline-terminated malformed journal record", () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-attempt-journal-")); + try { + const file = path.join(output, "attempt-journal.jsonl"); + const malformed = '{"raw":\n'; + fs.writeFileSync(file, malformed); + + expect(() => recoverAttemptJournal(output)).toThrow(/corrupt at line 1/u); + expect(fs.readFileSync(file, "utf8")).toBe(malformed); + } finally { + fs.rmSync(output, { recursive: true, force: true }); + } + }); + + it("requires two fresh campaigns with identical frozen inference configuration", () => { + const { manifest, attestations } = attestationFixture(); + expect(() => validateCampaignAttestations(manifest, attestations)).not.toThrow(); + + const reusedContainer = structuredClone(attestations); + reusedContainer[1].inference_container_id_sha256 = + reusedContainer[0].inference_container_id_sha256; + expect(() => validateCampaignAttestations(manifest, reusedContainer)).toThrow( + /distinct vLLM containers/u, + ); + + const changedConfiguration = structuredClone(attestations); + changedConfiguration[1].inference_config_sha256 = hex(101); + expect(() => validateCampaignAttestations(manifest, changedConfiguration)).toThrow( + /identical frozen config/u, + ); + + const reusedSandbox = structuredClone(attestations); + reusedSandbox[1].sandbox_cells[0].instance_id_sha256 = + reusedSandbox[0].sandbox_cells[0].instance_id_sha256; + expect(() => validateCampaignAttestations(manifest, reusedSandbox)).toThrow( + /reused or omitted sandbox instances/u, + ); + }); +}); diff --git a/test/performance/tool-disclosure-fixture.test.ts b/test/performance/tool-disclosure-fixture.test.ts new file mode 100644 index 0000000000..5598e71e7c --- /dev/null +++ b/test/performance/tool-disclosure-fixture.test.ts @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { assembleToolDisclosureRun } from "../../scripts/performance/tool-disclosure/assemble-run"; +import { + canonicalJson, + executeSyntheticTool, + generateSyntheticCatalog, + sha256Hex, +} from "../../scripts/performance/tool-disclosure/catalog"; +import { gradeTaskRun } from "../../scripts/performance/tool-disclosure/grading"; +import { + assertDockerfileSandboxBase, + writeOpenClawFixture, +} from "../../scripts/performance/tool-disclosure/openclaw-fixture"; +import { buildToolDisclosureSchedule } from "../../scripts/performance/tool-disclosure/schedule"; +import { buildPrimaryTasks } from "../../scripts/performance/tool-disclosure/tasks"; +import { + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + type ToolDisclosureManifest, +} from "../../scripts/performance/tool-disclosure/types"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + delete process.env.NEMOCLAW_PERFORMANCE_TEST_CALLS_PATH; + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +describe("OpenClaw performance test fixture", () => { + it("registers and executes the deterministic native tool catalog", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-performance-test-fixture-")); + temporaryDirectories.push(root); + const output = path.join(root, "fixture-16"); + const catalog = generateSyntheticCatalog({ size: 16 }); + writeOpenClawFixture({ + outputDir: output, + catalog, + sandboxBase: `registry.example/base@sha256:${"1".repeat(64)}`, + }); + + const registered: Array> = []; + const callsPath = path.join(root, "calls.jsonl"); + process.env.NEMOCLAW_PERFORMANCE_TEST_CALLS_PATH = callsPath; + const moduleUrl = `${pathToFileURL(path.join(output, "plugin", "index.js")).href}?test=1`; + const plugin = (await import(moduleUrl)) as { + default(api: { registerTool(tool: Record): void }): void; + }; + plugin.default({ registerTool: (tool) => registered.push(tool) }); + expect(registered).toHaveLength(16); + + const first = registered[0] as { + execute(id: string, params: Record): Promise<{ details: unknown }>; + }; + const args = buildPrimaryTasks(generateSyntheticCatalog({ size: 64 }))[0].expected_calls[0] + .arguments; + const result = await first.execute("call-1", args); + expect(result.details).toEqual(executeSyntheticTool(catalog.tools[0], args)); + const call = JSON.parse(fs.readFileSync(callsPath, "utf8")) as Record; + expect(call).toMatchObject({ + tool_name: catalog.tools[0].definition.function.name, + arguments_sha256: sha256Hex(canonicalJson(args)), + success: true, + }); + + const dockerfile = fs.readFileSync(path.join(output, "Dockerfile"), "utf8"); + expect(dockerfile).toContain("ARG NEMOCLAW_TOOL_DISCLOSURE=progressive"); + expect(dockerfile).toContain("ENV NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE}"); + const manifest = JSON.parse( + fs.readFileSync(path.join(output, "plugin", "openclaw.plugin.json"), "utf8"), + ) as { contracts: { tools: string[] } }; + expect(manifest.contracts.tools).toHaveLength(16); + }); + + it("rejects Dockerfile injection before creating the fixture", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-performance-test-fixture-")); + temporaryDirectories.push(root); + const output = path.join(root, "fixture-injected"); + const injected = `registry.example/base\nRUN touch /tmp/injected\n#@sha256:${"1".repeat(64)}`; + + expect(() => + writeOpenClawFixture({ + outputDir: output, + catalog: generateSyntheticCatalog({ size: 16 }), + sandboxBase: injected, + }), + ).toThrow(/canonical registry\/repository reference/u); + expect(fs.existsSync(output)).toBe(false); + }); + + it("rejects a generated Dockerfile base mismatch", () => { + const expected = `registry.example/base@sha256:${"1".repeat(64)}`; + const substituted = `registry.example/base@sha256:${"2".repeat(64)}`; + expect(() => + assertDockerfileSandboxBase( + `ARG SANDBOX_BASE=${substituted}\nFROM \${SANDBOX_BASE}\n`, + expected, + ), + ).toThrow(/does not match/u); + expect(() => + assertDockerfileSandboxBase( + `ARG SANDBOX_BASE=${expected}\nFROM registry.example/other@sha256:${"3".repeat(64)}\n`, + expected, + ), + ).toThrow(/does not match/u); + }); +}); + +describe("performance test grading", () => { + it("requires exact ordered calls, argument hashes, nonces, and final oracle", () => { + const catalog = generateSyntheticCatalog({ size: 64 }); + const task = buildPrimaryTasks(catalog).find((candidate) => candidate.kind === "ordered-chain"); + expect(task).toBeDefined(); + const requiredTask = task as NonNullable; + const calls = requiredTask.expected_calls.map((call) => ({ + tool_name: call.tool_name, + arguments_sha256: sha256Hex(canonicalJson(call.arguments)), + result_nonce: call.result_nonce, + success: true, + })); + const passing = gradeTaskRun( + requiredTask, + calls, + requiredTask.expected_final_includes.join(" "), + ); + expect(passing.outcome).toBe("success"); + expect( + gradeTaskRun( + requiredTask, + [...calls].reverse(), + requiredTask.expected_final_includes.join(" "), + ), + ).toMatchObject({ outcome: "incorrect", correctness: { expected_tool_order: false } }); + }); + + it("passes a no-tool control only when no calls occur and the phrase is present", () => { + const task = buildPrimaryTasks(generateSyntheticCatalog({ size: 64 })).find( + (candidate) => candidate.kind === "no-tool", + ); + expect(task).toBeDefined(); + const requiredTask = task as NonNullable; + expect(gradeTaskRun(requiredTask, [], requiredTask.expected_final_includes[0]).outcome).toBe( + "success", + ); + expect( + gradeTaskRun( + requiredTask, + [ + { + tool_name: "unexpected", + arguments_sha256: "0".repeat(64), + result_nonce: null, + success: true, + }, + ], + requiredTask.expected_final_includes[0], + ).outcome, + ).toBe("incorrect"); + }); + + it("assembles public-safe success and terminal run outcomes", () => { + const catalog = generateSyntheticCatalog({ size: 64 }); + const tasks = buildPrimaryTasks(catalog); + const task = tasks[0]; + const scheduled = buildToolDisclosureSchedule({ + primaryTaskIds: tasks.map((item) => item.id), + stressTaskIds: Array.from({ length: 8 }, (_, index) => `stress-${index}`), + seed: 1, + campaigns: [1], + }).find((run) => run.task_id === task.id && run.agent === "openclaw"); + expect(scheduled).toBeDefined(); + const requiredScheduled = scheduled as NonNullable; + const manifest = { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: "fixture-performance-test", + campaigns: [ + { + campaign_id: "campaign-1", + ordinal: 1, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + ], + protocol: { execution_seed: 1 }, + } as unknown as ToolDisclosureManifest; + const expected = task.expected_calls[0]; + const recordedCalls = [ + { + tool_name: expected.tool_name, + arguments_sha256: sha256Hex(canonicalJson(expected.arguments)), + result_nonce: expected.result_nonce, + success: true, + }, + ] as const; + const recorderEvents = [ + { + run_id: requiredScheduled.run_id, + request_sequence: 1, + model_call_sequence: 1, + endpoint: "chat-completions", + method: "POST", + visible_tool_count: 4, + canonical_tools_json_bytes: 321, + tools_sha256: "a".repeat(64), + tool_names: [expected.tool_name], + streaming: true, + status_code: 200, + started_monotonic_ms: 1, + first_byte_monotonic_ms: 2, + ended_monotonic_ms: 3, + duration_ms: 2, + time_to_first_byte_ms: 1, + outcome: "completed", + error_reason: null, + }, + ] as const; + const record = assembleToolDisclosureRun({ + manifest, + scheduled: requiredScheduled, + task, + calls: recordedCalls, + recorderEvents, + invocation: { + exit_code: 0, + timed_out: false, + elapsed_ms: 10, + final_output: `private output ${expected.result_nonce}`, + }, + initialSchemaTokens: 42, + }); + expect(record.outcome).toBe("success"); + expect(record.measurements.initial_tool_schema).toEqual({ + tool_count: 4, + serialized_bytes: 321, + tokenizer_tokens: 42, + }); + expect(JSON.stringify(record)).not.toContain("private output"); + expect(JSON.stringify(record)).not.toContain(String(expected.arguments.resource_id)); + + for (const scenario of [ + { + expected: "timeout", + invocation: { exit_code: null, timed_out: true, elapsed_ms: 10, final_output: "" }, + failureOutcome: undefined, + }, + { + expected: "model-error", + invocation: { exit_code: 7, timed_out: false, elapsed_ms: 10, final_output: "" }, + failureOutcome: undefined, + }, + { + expected: "context-overflow", + invocation: { exit_code: 7, timed_out: false, elapsed_ms: 10, final_output: "" }, + failureOutcome: "context-overflow", + }, + ] as const) { + const terminal = assembleToolDisclosureRun({ + manifest, + scheduled: requiredScheduled, + task, + calls: recordedCalls, + recorderEvents, + invocation: scenario.invocation, + initialSchemaTokens: 42, + ...(scenario.failureOutcome ? { failureOutcome: scenario.failureOutcome } : {}), + }); + expect(terminal.outcome).toBe(scenario.expected); + expect(terminal.scored).toBe(true); + } + + const staticScheduled = buildToolDisclosureSchedule({ + primaryTaskIds: tasks.map((item) => item.id), + stressTaskIds: Array.from({ length: 8 }, (_, index) => `stress-${index}`), + seed: 1, + campaigns: [1], + }).find((run) => run.phase === "static-visibility"); + expect(staticScheduled).toBeDefined(); + const requiredStatic = staticScheduled as NonNullable; + const staticFailure = assembleToolDisclosureRun({ + manifest, + scheduled: requiredStatic, + calls: [], + recorderEvents: [{ ...recorderEvents[0], run_id: requiredStatic.run_id }], + invocation: { exit_code: 0, timed_out: false, elapsed_ms: 10, final_output: "" }, + initialSchemaTokens: 0, + }); + expect(staticFailure.outcome).toBe("model-error"); + }); +}); diff --git a/test/performance/tool-disclosure-mcp-server.test.ts b/test/performance/tool-disclosure-mcp-server.test.ts new file mode 100644 index 0000000000..eeac1adbdf --- /dev/null +++ b/test/performance/tool-disclosure-mcp-server.test.ts @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildSyntheticArguments, + generateSyntheticCatalog, +} from "../../scripts/performance/tool-disclosure/catalog"; +import { + executeArgumentHash, + SyntheticMcpServer, +} from "../../scripts/performance/tool-disclosure/mcp-server"; + +async function rpc( + url: string, + token: string, + method: string, + params: Record = {}, +): Promise { + return await fetch(url, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }), + }); +} + +describe("synthetic performance test MCP server", () => { + it("lists and executes the generated catalog while retaining content-free call evidence", async () => { + const catalog = generateSyntheticCatalog({ size: 16, seed: "mcp-server-test" }); + const token = "fixture-token-that-must-not-be-recorded"; + const server = new SyntheticMcpServer(catalog, token); + const address = await server.start(); + try { + const unauthorized = await fetch(address.local_url, { method: "POST", body: "{}" }); + expect(unauthorized.status).toBe(401); + + const initialized = await rpc(address.local_url, token, "initialize", { + protocolVersion: "2025-03-26", + }); + expect((await initialized.json()) as object).toMatchObject({ + result: { serverInfo: { name: "nemoclaw-tool-disclosure-performance-test" } }, + }); + + const listed = await rpc(address.local_url, token, "tools/list"); + const listPayload = (await listed.json()) as { result: { tools: unknown[] } }; + expect(listPayload.result.tools).toHaveLength(16); + + const tool = catalog.tools[0]; + const args = buildSyntheticArguments(tool, 42); + server.beginRun("c1-primary-openclaw-progressive"); + const called = await rpc(address.local_url, token, "tools/call", { + name: tool.definition.function.name, + arguments: args, + }); + const callPayload = (await called.json()) as { result: { content: Array<{ text: string }> } }; + expect(callPayload.result.content[0].text).toContain('"nonce"'); + const events = server.endRun(); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + run_id: "c1-primary-openclaw-progressive", + sequence: 1, + tool_name: tool.definition.function.name, + arguments_sha256: executeArgumentHash(args), + success: true, + }); + expect(JSON.stringify(events)).not.toContain(token); + expect(JSON.stringify(events)).not.toContain(JSON.stringify(args)); + } finally { + await server.stop(); + } + }); + + it("rejects a bearer token reused from another campaign", async () => { + const server = new SyntheticMcpServer( + generateSyntheticCatalog({ size: 16, seed: "mcp-token-rotation-test" }), + "current-campaign-token", + ); + const address = await server.start(); + try { + const response = await rpc(address.local_url, "previous-campaign-token", "tools/list"); + expect(response.status).toBe(401); + } finally { + await server.stop(); + } + }); +}); diff --git a/test/performance/tool-disclosure-quick-tunnel.test.ts b/test/performance/tool-disclosure-quick-tunnel.test.ts new file mode 100644 index 0000000000..c06dbaf1fa --- /dev/null +++ b/test/performance/tool-disclosure-quick-tunnel.test.ts @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + buildQuickTunnelArgs, + buildQuickTunnelEnvironment, + parseQuickTunnelOrigin, + startQuickTunnel, +} from "../../scripts/performance/tool-disclosure/quick-tunnel"; + +describe("tool-disclosure quick tunnel", () => { + it("extracts only a bounded trycloudflare origin", () => { + expect( + parseQuickTunnelOrigin( + "INF Requesting new quick Tunnel https://performance-test-123.trycloudflare.com", + ), + ).toBe("https://performance-test-123.trycloudflare.com"); + expect(parseQuickTunnelOrigin("https://example.com/secret")).toBeNull(); + }); + + it("uses the latest origin when cloudflared reports more than one", () => { + expect( + parseQuickTunnelOrigin( + "old https://stale-123.trycloudflare.com\nnew https://current-456.trycloudflare.com\n", + ), + ).toBe("https://current-456.trycloudflare.com"); + }); + + it("omits ambient Cloudflare account configuration from the subprocess", () => { + expect( + buildQuickTunnelEnvironment( + { + PATH: "/usr/bin", + HOME: "/home/operator", + XDG_CONFIG_HOME: "/home/operator/.config", + LC_ALL: "C.UTF-8", + }, + "/tmp/isolated-cloudflared-home", + ), + ).toEqual({ + PATH: "/usr/bin", + LC_ALL: "C.UTF-8", + HOME: "/tmp/isolated-cloudflared-home", + XDG_CONFIG_HOME: "/tmp/isolated-cloudflared-home", + }); + }); + + it("builds a loopback origin command and rejects invalid ports", () => { + expect(buildQuickTunnelArgs(31337)).toEqual([ + "tunnel", + "--config=", + "--no-autoupdate", + "--protocol", + "http2", + "--url", + "http://127.0.0.1:31337", + "--loglevel", + "info", + ]); + expect(() => buildQuickTunnelArgs(0)).toThrow("between 1 and 65535"); + expect(buildQuickTunnelArgs(31337, "auto")).toContain("auto"); + }); + + it("discovers a tunnel origin from the bounded child log and closes the child", async () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-quick-tunnel-test-")); + const binary = path.join(fixture, "fake-cloudflared"); + fs.writeFileSync( + binary, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "printf '%s\\n' 'INF Requesting new quick Tunnel https://log-backed-123.trycloudflare.com' >&2", + "trap 'exit 0' TERM INT", + "while :; do sleep 1; done", + "", + ].join("\n"), + { mode: 0o755 }, + ); + + try { + const tunnel = await startQuickTunnel({ + port: 31_337, + binary, + env: { PATH: process.env.PATH }, + fetchImpl: (async (input) => { + expect(String(input)).toBe("https://log-backed-123.trycloudflare.com/mcp"); + return new Response(null, { status: 405 }); + }) as typeof fetch, + timeoutMs: 5_000, + }); + expect(tunnel.mcpUrl).toBe("https://log-backed-123.trycloudflare.com/mcp"); + await tunnel.close(); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); +}); diff --git a/test/performance/tool-disclosure-recorder.test.ts b/test/performance/tool-disclosure-recorder.test.ts new file mode 100644 index 0000000000..a057c366d3 --- /dev/null +++ b/test/performance/tool-disclosure-recorder.test.ts @@ -0,0 +1,962 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import http, { type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo, Socket } from "node:net"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { canonicalJson, type JsonValue } from "../../scripts/performance/tool-disclosure/catalog"; +import { + createToolDisclosureRecordingProxy, + type RecordingProxyRequestTransform, + type ToolDisclosureRecordingProxy, +} from "../../scripts/performance/tool-disclosure/recorder"; + +interface TestUpstream { + baseUrl: string; + close: () => Promise; +} + +const cleanup: Array<() => Promise> = []; + +afterEach(async () => { + await Promise.allSettled(cleanup.splice(0).map((close) => close())); +}); + +async function startUpstream( + handler: (request: IncomingMessage, response: ServerResponse) => void, +): Promise { + const sockets = new Set(); + const server = http.createServer(handler); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + const address = server.address(); + expect(address).not.toBeNull(); + expect(typeof address).not.toBe("string"); + const tcpAddress = address as AddressInfo; + + const close = async (): Promise => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + }; + cleanup.push(close); + return { baseUrl: `http://127.0.0.1:${tcpAddress.port}`, close }; +} + +async function startProxy( + upstreamBaseUrl: string, + options: { + listenHost?: string; + allowAuthenticatedPrivateIpv4Listener?: boolean; + requiredAuthorization?: string; + upstreamAuthorization?: string; + maxRequestBodyBytes?: number; + requestTimeoutMs?: number; + requiredTemperature?: number; + requestTransform?: RecordingProxyRequestTransform; + } = {}, +): Promise<{ proxy: ToolDisclosureRecordingProxy; baseUrl: string }> { + const proxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl, + ...options, + }); + const address = await proxy.start(); + cleanup.push(() => proxy.stop()); + expect(address.host).toBe("127.0.0.1"); + expect(address.base_url).toBe(`http://127.0.0.1:${address.port}`); + return { proxy, baseUrl: address.base_url }; +} + +function collectBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + request.once("error", reject); + }); +} + +describe("tool-disclosure recording proxy", () => { + it("fails closed for non-loopback listeners and credential-bearing upstream URLs", () => { + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://localhost:8000/v1", + listenHost: "0.0.0.0", + }), + ).toThrow("must be exactly 127.0.0.1"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://user:password@localhost:8000/v1", + }), + ).toThrow("must not contain credentials"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://localhost:8000/v1?api_key=private-key", + }), + ).toThrow("must not contain a query or fragment"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://inference.example/v1", + }), + ).toThrow("upstreamBaseUrl is allowed only on loopback"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + listenHost: "172.18.0.1", + allowAuthenticatedPrivateIpv4Listener: true, + }), + ).toThrow("authenticated private IPv4 listener"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + listenHost: "203.0.113.10", + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: "Bearer ingress-token", + upstreamAuthorization: "Bearer upstream-token", + }), + ).toThrow("must be exactly 127.0.0.1"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + listenHost: "172.18.0.1", + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: "Bearer ingress-token", + upstreamAuthorization: "Bearer upstream-token", + }), + ).not.toThrow(); + }); + + it("requires paired, bounded bearer headers for authenticated private listeners", () => { + const privateListener = { + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + listenHost: "172.18.0.1", + allowAuthenticatedPrivateIpv4Listener: true, + } as const; + expect(() => + createToolDisclosureRecordingProxy({ + ...privateListener, + requiredAuthorization: "Bearer ingress-token", + }), + ).toThrow("requires both headers"); + expect(() => + createToolDisclosureRecordingProxy({ + ...privateListener, + upstreamAuthorization: "Bearer upstream-token", + }), + ).toThrow("requires both headers"); + for (const malformed of [ + "Basic token", + "Bearer ", + "Bearer token with spaces", + `Bearer ${"x".repeat(4_097)}`, + ]) { + expect(() => + createToolDisclosureRecordingProxy({ + ...privateListener, + requiredAuthorization: malformed, + upstreamAuthorization: "Bearer upstream-token", + }), + ).toThrow("bounded Bearer authorization header"); + } + }); + + it("accepts only RFC1918 address ranges for authenticated private listeners", () => { + const options = { + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + allowAuthenticatedPrivateIpv4Listener: true, + requiredAuthorization: "Bearer ingress-token", + upstreamAuthorization: "Bearer upstream-token", + } as const; + for (const listenHost of ["10.0.0.1", "172.16.0.1", "172.31.255.254", "192.168.1.1"]) { + expect(() => createToolDisclosureRecordingProxy({ ...options, listenHost })).not.toThrow(); + } + for (const listenHost of ["172.15.255.254", "172.32.0.1", "192.167.1.1", "192.169.1.1"]) { + expect(() => createToolDisclosureRecordingProxy({ ...options, listenHost })).toThrow( + "must be exactly 127.0.0.1", + ); + } + }); + + it("rejects non-loopback HTTPS recorder upstreams for the local performance test", () => { + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + }), + ).toThrow("upstreamBaseUrl is allowed only on loopback"); + for (const upstreamBaseUrl of [ + "http://localhost:8000/v1", + "https://127.0.0.1:8000/v1", + "https://[::1]:8000/v1", + ]) { + expect(() => createToolDisclosureRecordingProxy({ upstreamBaseUrl })).not.toThrow(); + } + }); + + it("requires an explicit opt-in and HTTPS for a remote recording upstream", () => { + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://inference.example/v1", + allowRemoteHttpsUpstream: true, + }), + ).not.toThrow(); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://inference.example/v1", + allowRemoteHttpsUpstream: true, + }), + ).toThrow("a remote recording upstream must use HTTPS"); + }); + + it("canonicalizes the IPv6 loopback and rejects mapped or non-loopback variants", () => { + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://[0:0:0:0:0:0:0:1]:8000/v1", + }), + ).not.toThrow(); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://[::ffff:127.0.0.1]:8000/v1", + }), + ).toThrow("upstreamBaseUrl is allowed only on loopback"); + expect(() => + createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "http://[::2]:8000/v1", + }), + ).toThrow("upstreamBaseUrl is allowed only on loopback"); + }); + + it("normalizes localhost to literal IPv4 before connecting upstream", async () => { + let observedHost = ""; + let observedPath = ""; + const upstream = await startUpstream((request, response) => { + observedHost = request.headers.host ?? ""; + observedPath = request.url ?? ""; + response.end("{}"); + }); + const upstreamUrl = new URL(upstream.baseUrl); + upstreamUrl.hostname = "localhost"; + upstreamUrl.pathname = "/proxy/v1"; + const { proxy, baseUrl } = await startProxy(upstreamUrl.toString()); + + proxy.beginRun("localhost-normalization"); + const response = await fetch(`${baseUrl}/v1/models`); + await response.arrayBuffer(); + proxy.endRun(); + + expect(response.status).toBe(200); + expect(observedHost).toBe(`127.0.0.1:${upstreamUrl.port}`); + expect(observedPath).toBe("/proxy/v1/models"); + }); + + it("accepts bounded public scheduled run IDs and rejects unsafe IDs", () => { + const proxy = createToolDisclosureRecordingProxy({ + upstreamBaseUrl: "https://localhost:8000/v1", + }); + const scheduledId = "c1--primary--openclaw--progressive--n512--single-01--r1"; + expect(proxy.beginRun(scheduledId)).toBe(scheduledId); + expect(proxy.endRun()).toEqual([]); + + expect(() => proxy.beginRun("run id with prompt content")).toThrow("public-safe"); + expect(() => proxy.beginRun(`r${"x".repeat(256)}`)).toThrow("1-256"); + }); + + it("forwards requests while retaining only canonical tool metadata", async () => { + const received: Array<{ + url: string; + authorization: string; + body: string; + }> = []; + const upstream = await startUpstream((request, response) => { + void collectBody(request).then((body) => { + received.push({ + url: request.url ?? "", + authorization: String(request.headers.authorization ?? ""), + body, + }); + response.writeHead(200, { + "content-type": "application/json", + "x-private-upstream-header": "header-secret", + }); + response.end('{"choices":[{"message":{"content":"response-secret"}}]}'); + }); + }); + const { proxy, baseUrl } = await startProxy(`${upstream.baseUrl}/v1`); + const tool = { + type: "function", + function: { + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + name: "weather_lookup", + description: "schema-secret", + }, + }; + const requestBody = JSON.stringify({ + model: "model-secret", + messages: [{ role: "user", content: "prompt-secret" }], + tools: [tool], + stream: false, + }); + + const runId = proxy.beginRun(); + const firstResponse = await fetch(`${baseUrl}/v1/chat/completions?api_key=query-secret`, { + method: "POST", + headers: { + authorization: "Bearer authorization-secret", + "content-type": "application/json", + }, + body: requestBody, + }); + expect(await firstResponse.text()).toContain("response-secret"); + + const reorderedTool = { + function: { + description: "schema-secret", + name: "weather_lookup", + parameters: { + properties: { city: { type: "string" } }, + type: "object", + }, + }, + type: "function", + }; + const secondResponse = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [], tools: [reorderedTool] }), + }); + await secondResponse.arrayBuffer(); + + const events = proxy.endRun(); + expect(events).toHaveLength(2); + expect(events[0]).toMatchObject({ + run_id: runId, + request_sequence: 1, + model_call_sequence: 1, + endpoint: "chat-completions", + method: "POST", + visible_tool_count: 1, + tool_names: ["weather_lookup"], + status_code: 200, + outcome: "completed", + error_reason: null, + }); + expect(events[1].model_call_sequence).toBe(2); + expect(events[1].tools_sha256).toBe(events[0].tools_sha256); + + const canonical = JSON.stringify([ + { + function: { + description: "schema-secret", + name: "weather_lookup", + parameters: { + properties: { city: { type: "string" } }, + type: "object", + }, + }, + type: "function", + }, + ]); + expect(events[0].canonical_tools_json_bytes).toBe(Buffer.byteLength(canonical)); + expect(events[0].tools_sha256).toBe(createHash("sha256").update(canonical).digest("hex")); + + expect(received[0]).toEqual({ + url: "/v1/chat/completions?api_key=query-secret", + authorization: "Bearer authorization-secret", + body: requestBody, + }); + const serializedEvents = JSON.stringify(events); + for (const secret of [ + "prompt-secret", + "model-secret", + "schema-secret", + "authorization-secret", + "query-secret", + "response-secret", + "header-secret", + ]) { + expect(serializedEvents).not.toContain(secret); + } + + const snapshots = proxy.consumeToolSchemaSnapshots(runId); + expect(snapshots).toHaveLength(2); + expect(snapshots[0]).toEqual({ + run_id: runId, + model_call_sequence: 1, + canonical_tools_json: canonical, + }); + expect(proxy.consumeToolSchemaSnapshots(runId)).toEqual([]); + + proxy.resetEvents(); + expect(proxy.getEvents()).toEqual([]); + }); + + it("authenticates before recording or transformation and replaces the upstream credential", async () => { + const upstreamAuthorizations: string[] = []; + const upstream = await startUpstream((request, response) => { + upstreamAuthorizations.push(String(request.headers.authorization ?? "")); + response.end("{}"); + }); + let transformCalls = 0; + const ingressAuthorization = "Bearer private-ingress-token"; + const upstreamAuthorization = "Bearer private-upstream-token"; + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requiredAuthorization: ingressAuthorization, + upstreamAuthorization, + requestTransform: ({ body }) => { + transformCalls += 1; + return body; + }, + }); + + proxy.beginRun("authorization-translation"); + const missing = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[],"tools":[]}', + }); + expect(missing.status).toBe(401); + expect(await missing.json()).toEqual({ error: "unauthorized" }); + const rejected = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { + authorization: "Bearer private-ingress-tokeN", + "content-type": "application/json", + }, + body: '{"messages":[],"tools":[]}', + }); + expect(rejected.status).toBe(401); + expect(await rejected.json()).toEqual({ error: "unauthorized" }); + expect(transformCalls).toBe(0); + expect(upstreamAuthorizations).toEqual([]); + + const acceptedBody = '{"messages":[],"tools":[]}'; + const acceptedStatus = await new Promise((resolve, reject) => { + const request = http.request(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { + authorization: ingressAuthorization, + connection: "authorization", + "content-length": Buffer.byteLength(acceptedBody), + "content-type": "application/json", + }, + }); + request.once("response", (response) => { + response.resume(); + response.once("end", () => resolve(response.statusCode ?? 0)); + }); + request.once("error", reject); + request.end(acceptedBody); + }); + expect(acceptedStatus).toBe(200); + + const events = proxy.endRun(); + expect(transformCalls).toBe(1); + expect(upstreamAuthorizations).toEqual([upstreamAuthorization]); + expect(events).toHaveLength(1); + expect(JSON.stringify(events)).not.toMatch(/private-ingress-token|private-upstream-token/); + }); + + it("applies an async request transform before forwarding and records transformed tools", async () => { + let receivedBody = ""; + const upstream = await startUpstream((request, response) => { + void collectBody(request).then((body) => { + receivedBody = body; + response.end("{}"); + }); + }); + const observedContexts: Array<{ + runId: string | null; + endpoint: string; + method: string; + modelCallSequence: number | null; + body: string; + }> = []; + const requestTransform: RecordingProxyRequestTransform = async (input) => { + observedContexts.push({ + runId: input.runId, + endpoint: input.endpoint, + method: input.method, + modelCallSequence: input.modelCallSequence, + body: input.body.toString("utf8"), + }); + const payload = JSON.parse(input.body.toString("utf8")) as Record; + const tools = payload.tools as unknown[]; + payload.tools = [tools[1]]; + await Promise.resolve(); + return Buffer.from(JSON.stringify(payload), "utf8"); + }; + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTransform, + }); + const requestBody = JSON.stringify({ + model: "private-model", + messages: [{ role: "user", content: "private-prompt" }], + tools: [ + { + type: "function", + function: { + name: "first_tool", + description: "first-private", + parameters: {}, + }, + }, + { + type: "function", + function: { + name: "second_tool", + description: "second-private", + parameters: {}, + }, + }, + ], + }); + + const runId = proxy.beginRun("transform-run"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: requestBody, + }); + expect(response.status).toBe(200); + await response.arrayBuffer(); + + const [event] = proxy.endRun(); + const expectedPayload = JSON.parse(requestBody) as Record; + expectedPayload.tools = [(expectedPayload.tools as unknown[])[1]]; + const expectedBody = JSON.stringify(expectedPayload); + const canonicalTools = canonicalJson(expectedPayload.tools as JsonValue); + expect(receivedBody).toBe(expectedBody); + expect(observedContexts).toEqual([ + { + runId, + endpoint: "chat-completions", + method: "POST", + modelCallSequence: 1, + body: requestBody, + }, + ]); + expect(event).toMatchObject({ + visible_tool_count: 1, + tool_names: ["second_tool"], + canonical_tools_json_bytes: Buffer.byteLength(canonicalTools), + tools_sha256: createHash("sha256").update(canonicalTools).digest("hex"), + status_code: 200, + outcome: "completed", + }); + expect(proxy.consumeToolSchemaSnapshots(runId)).toEqual([ + { + run_id: runId, + model_call_sequence: 1, + canonical_tools_json: canonicalTools, + }, + ]); + expect(JSON.stringify(event)).not.toMatch( + /private-model|private-prompt|first-private|second-private/, + ); + }); + + it("forwards request bytes unchanged when no transform is configured", async () => { + let receivedBody = Buffer.alloc(0); + let receivedLength = ""; + const upstream = await startUpstream((request, response) => { + receivedLength = String(request.headers["content-length"] ?? ""); + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.once("end", () => { + receivedBody = Buffer.concat(chunks); + response.end("{}"); + }); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl); + const exactBody = Buffer.from( + '{\n "messages" : [{"content":"byte-private"}],\n "tools" : []\n}\n', + "utf8", + ); + + proxy.beginRun("no-transform-bytes"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: exactBody, + }); + await response.arrayBuffer(); + proxy.endRun(); + + expect(response.status).toBe(200); + expect(receivedBody.equals(exactBody)).toBe(true); + expect(receivedLength).toBe(String(exactBody.length)); + }); + + it("rejects transform failures with a fixed content-free proxy error", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTransform: async ({ body }) => { + expect(body.toString("utf8")).toContain("transform-input-private"); + throw new Error("transform-error-private"); + }, + }); + + const runId = proxy.beginRun("transform-error"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[{"content":"transform-input-private"}]}', + }); + expect(response.status).toBe(502); + expect(await response.json()).toEqual({ error: "recording proxy failure" }); + expect(hits).toBe(0); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + visible_tool_count: 0, + status_code: 502, + outcome: "request-rejected", + error_reason: "proxy-failure", + }); + expect(proxy.consumeToolSchemaSnapshots(runId)).toEqual([]); + expect(JSON.stringify(event)).not.toMatch(/transform-input-private|transform-error-private/); + }); + + it("rejects a transformed body that exceeds the request size bound", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + maxRequestBodyBytes: 32, + requestTransform: () => Buffer.alloc(33, "x"), + }); + + proxy.beginRun("transform-too-large"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{}", + }); + expect(response.status).toBe(413); + expect(await response.json()).toEqual({ error: "request body too large" }); + expect(hits).toBe(0); + expect(proxy.endRun()[0]).toMatchObject({ + visible_tool_count: 0, + status_code: 413, + outcome: "request-rejected", + error_reason: "request-body-too-large", + }); + }); + + it("enforces the frozen temperature after transformation", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requiredTemperature: 0, + requestTransform: ({ body }) => { + const payload = JSON.parse(body.toString("utf8")) as Record; + payload.temperature = 0.5; + return Buffer.from(JSON.stringify(payload), "utf8"); + }, + }); + + proxy.beginRun("transform-temperature"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[],"tools":[]}', + }); + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: "performance test temperature mismatch", + }); + expect(hits).toBe(0); + expect(proxy.endRun()[0]).toMatchObject({ + status_code: 400, + outcome: "request-rejected", + error_reason: "proxy-failure", + }); + }); + + it("streams SSE bytes unchanged and records monotonic first-byte timing", async () => { + const chunks = [ + 'data: {"choices":[{"delta":{"content":"A"}}]}\n\n', + 'data: {"choices":[{"delta":{"content":"B"}}]}\n\n', + "data: [DONE]\n\n", + ]; + const upstream = await startUpstream((_request, response) => { + response.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + }); + response.write(chunks[0]); + setTimeout(() => { + response.write(chunks[1]); + response.end(chunks[2]); + }, 15); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/v1/responses`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "private-model", + input: "private-input", + stream: true, + tools: [{ type: "function", name: "calculator", parameters: {} }], + }), + }); + expect(response.headers.get("content-type")).toBe("text/event-stream"); + expect(await response.text()).toBe(chunks.join("")); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + endpoint: "responses", + visible_tool_count: 1, + tool_names: ["calculator"], + status_code: 200, + outcome: "completed", + }); + expect(event.first_byte_monotonic_ms).not.toBeNull(); + expect(event.started_monotonic_ms).toBeLessThanOrEqual(event.first_byte_monotonic_ms ?? 0); + expect(event.first_byte_monotonic_ms ?? Number.POSITIVE_INFINITY).toBeLessThanOrEqual( + event.ended_monotonic_ms, + ); + expect(event.duration_ms).toBeGreaterThanOrEqual(event.time_to_first_byte_ms ?? 0); + }); + + it("forces a missing frozen temperature and rejects a conflicting value", async () => { + const received: string[] = []; + const upstream = await startUpstream((request, response) => { + void collectBody(request).then((body) => { + received.push(body); + response.end("{}"); + }); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requiredTemperature: 0, + }); + proxy.beginRun("temperature-missing"); + await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [], tools: [] }), + }); + proxy.endRun(); + expect(JSON.parse(received[0])).toMatchObject({ temperature: 0 }); + + proxy.beginRun("temperature-conflict"); + const rejected = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [], tools: [], temperature: 0.5 }), + }); + expect(rejected.status).toBe(400); + proxy.endRun(); + expect(received).toHaveLength(1); + }); + + it("rejects upstream redirects without relaying Location or body secrets", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.writeHead(307, { + location: "https://example.invalid/v1?token=location-secret", + "content-type": "text/plain", + }); + response.end("redirect-response-secret"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[{"content":"request-secret"}]}', + }); + expect(response.status).toBe(502); + expect(response.headers.get("location")).toBeNull(); + expect(await response.json()).toEqual({ + error: "upstream redirect rejected", + }); + expect(hits).toBe(1); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + status_code: 502, + first_byte_monotonic_ms: null, + outcome: "request-rejected", + error_reason: "proxy-failure", + }); + expect(JSON.stringify(event)).not.toMatch( + /location-secret|redirect-response-secret|request-secret/, + ); + }); + + it("rejects oversized bodies without contacting the upstream", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + maxRequestBodyBytes: 32, + }); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ messages: [{ content: "x".repeat(100) }] }), + }); + expect(response.status).toBe(413); + expect(await response.json()).toEqual({ error: "request body too large" }); + expect(hits).toBe(0); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + status_code: 413, + outcome: "request-rejected", + error_reason: "request-body-too-large", + visible_tool_count: 0, + }); + }); + + it("bounds stalled upstream calls and exposes only a redacted error class", async () => { + const upstream = await startUpstream(async (request, response) => { + await collectBody(request); + setTimeout(() => { + response.writeHead(500, { "content-type": "text/plain" }); + response.end("upstream-error-body-secret"); + }, 200); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTimeoutMs: 30, + }); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { + authorization: "Bearer timeout-authorization-secret", + "content-type": "application/json", + }, + body: '{"messages":[{"content":"timeout-prompt-secret"}]}', + }); + expect(response.status).toBe(504); + expect(await response.json()).toEqual({ error: "upstream timeout" }); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + status_code: 504, + outcome: "upstream-timeout", + error_reason: "upstream-timeout", + }); + expect(JSON.stringify(event)).not.toMatch( + /timeout-authorization-secret|timeout-prompt-secret|upstream-error-body-secret/, + ); + }); + + it("aborts a stalled request transform before an upstream call begins", async () => { + let upstreamHits = 0; + let transformAborted = false; + const upstream = await startUpstream((_request, response) => { + upstreamHits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTimeoutMs: 30, + requestTransform: ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => { + transformAborted = true; + reject(new DOMException("aborted", "AbortError")); + }, + { once: true }, + ); + }), + }); + + proxy.beginRun("transform-timeout"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"messages":[],"tools":[]}', + }); + + expect(response.status).toBe(408); + expect(await response.json()).toEqual({ error: "request timeout" }); + expect(transformAborted).toBe(true); + expect(upstreamHits).toBe(0); + expect(proxy.endRun()[0]).toMatchObject({ + status_code: 408, + outcome: "request-rejected", + error_reason: "request-timeout", + }); + }); + + it("bounds an SSE response that stalls after its first byte", async () => { + const upstream = await startUpstream((_request, response) => { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.write("data: first\n\n"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl, { + requestTimeoutMs: 30, + }); + + proxy.beginRun("c1--primary--hermes--progressive--n512--single-01--r1"); + const response = await fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: '{"stream":true}', + }); + expect(response.status).toBe(200); + await expect(response.text()).rejects.toBeDefined(); + + const [event] = proxy.endRun(); + expect(event).toMatchObject({ + status_code: 200, + outcome: "upstream-timeout", + error_reason: "upstream-timeout", + }); + expect(event.first_byte_monotonic_ms).not.toBeNull(); + }); + + it("does not forward or record paths outside the OpenAI v1 boundary", async () => { + let hits = 0; + const upstream = await startUpstream((_request, response) => { + hits += 1; + response.end("unexpected"); + }); + const { proxy, baseUrl } = await startProxy(upstream.baseUrl); + + proxy.beginRun(); + const response = await fetch(`${baseUrl}/health?token=health-secret`); + expect(response.status).toBe(404); + expect(hits).toBe(0); + expect(proxy.endRun()).toEqual([]); + }); +}); diff --git a/test/performance/tool-disclosure-report.test.ts b/test/performance/tool-disclosure-report.test.ts new file mode 100644 index 0000000000..e95a5d2565 --- /dev/null +++ b/test/performance/tool-disclosure-report.test.ts @@ -0,0 +1,486 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { renderToolDisclosureMarkdown } from "../../scripts/performance/tool-disclosure/report"; +import { + buildComparisonCells, + buildConservativeClaims, + buildToolDisclosureSummary, + evaluateClaimGates, + pairedBootstrapDifference, + summarizeStaticVisibility, +} from "../../scripts/performance/tool-disclosure/statistics"; +import { + type PerformanceTestPhase, + DEFAULT_BOOTSTRAP_SAMPLES, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + type ToolDisclosureAgent, + type ToolDisclosureManifest, + type ToolDisclosureMode, + type ToolDisclosureRun, +} from "../../scripts/performance/tool-disclosure/types"; + +const AGENTS = ["langchain-deepagents-code", "hermes", "openclaw"] as const; +const CAMPAIGNS = ["campaign-1", "campaign-2"] as const; +const TASK_IDS = Array.from({ length: 24 }, (_, index) => `task-${index + 1}`); + +function makeManifest(overrides: Partial = {}): ToolDisclosureManifest { + return { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: "tool-disclosure-example", + created_at: "2026-07-06T12:00:00.000Z", + sut: { git_sha: "3a05b54e", git_ref: "v0.0.74", worktree_clean: true }, + harness: { + git_sha: "1234abcd", + git_ref: "performance-test-harness", + worktree_clean: true, + }, + campaigns: [ + { + campaign_id: "campaign-1", + ordinal: 1, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + { + campaign_id: "campaign-2", + ordinal: 2, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + ], + protocol: { + agents: AGENTS, + modes: ["direct", "progressive"], + catalog_sizes: [512], + primary_catalog_size: 512, + tasks: TASK_IDS.map((taskId) => ({ + task_id: taskId, + kind: "single-tool" as const, + })), + repetitions: { "small-control": 1, primary: 1, "large-stress": 1 }, + execution_seed: 7, + bootstrap_samples: 500, + bootstrap_seed: 42, + noninferiority_margin_percentage_points: -5, + retry_setup_failures: 2, + }, + environment: { + operating_system: "Ubuntu 24.04", + architecture: "x86_64", + cpu_model: "Example CPU", + cpu_count: 20, + ram_gib: 128, + accelerator_type: "example-accelerator", + accelerator_model: "Example Accelerator", + accelerator_architecture: "example-architecture", + accelerator_count: 1, + accelerator_driver_version: "999.1", + accelerator_runtime: "example-runtime-1.0", + power_state: "maximum-performance", + openshell_version: "0.1.0", + agent_versions: { + "langchain-deepagents-code": "1.0.0", + hermes: "1.0.0", + openclaw: "1.0.0", + }, + }, + inference: { + api: "chat-completions", + model_id: "deepseek-ai/DeepSeek-V4-Flash", + model_revision: "revision-1", + container_image: "registry.example/vllm:1.0.0", + container_digest: "sha256:container", + vllm_version: "0.1.0", + tool_call_parser: "deepseek_v3", + reasoning_parser: "deepseek_r1", + temperature: 0, + concurrency: 1, + prefix_caching_enabled: true, + public_vllm_flags: ["--enable-prefix-caching"], + }, + ...overrides, + }; +} + +interface RunOptions { + campaign?: string; + agent?: ToolDisclosureAgent; + mode?: ToolDisclosureMode; + phase?: PerformanceTestPhase; + taskId?: string; + repetition?: number; + success?: boolean; + scored?: boolean; + schemaTokens?: number; + schemaBytes?: number; + visibleTools?: number; + totalPromptTokens?: number; + latencyMs?: number; +} + +let runSequence = 0; + +function makeRun(options: RunOptions = {}): ToolDisclosureRun { + const success = options.success ?? true; + const schemaTokens = options.schemaTokens ?? 1_000; + const mode = options.mode ?? "direct"; + const phase = options.phase ?? "primary"; + const scored = options.scored ?? phase !== "static-visibility"; + runSequence += 1; + return { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: "tool-disclosure-example", + campaign_id: options.campaign ?? "campaign-1", + run_id: `run-${runSequence}`, + phase, + agent: options.agent ?? "langchain-deepagents-code", + mode, + catalog_size: 512, + task_id: options.taskId ?? "task-1", + task_kind: "single-tool", + repetition: options.repetition ?? 1, + execution_seed: 7, + outcome: + phase === "static-visibility" + ? "success" + : scored + ? success + ? "success" + : "incorrect" + : "setup-error", + scored, + correctness: { + task_success: scored && success, + expected_tool_names: scored && success, + expected_tool_order: scored && success, + expected_arguments: scored && success, + expected_call_count: scored && success, + nonce_present: scored && success, + unnecessary_tool_calls: 0, + }, + measurements: { + initial_tool_schema: { + tool_count: options.visibleTools ?? (mode === "direct" ? 512 : 3), + serialized_bytes: options.schemaBytes ?? schemaTokens * 4, + tokenizer_tokens: schemaTokens, + }, + total_prompt_tokens: options.totalPromptTokens ?? schemaTokens + 100, + completion_tokens: 20, + time_to_first_response_byte_ms: (options.latencyMs ?? 200) / 2, + inference_time_ms: options.latencyMs ?? 200, + end_to_end_time_ms: options.latencyMs ?? 200, + model_calls: 1, + discovery_calls: mode === "progressive" ? 1 : 0, + }, + }; +} + +function completeEvidence(progressiveSchemaTokens = 10): ToolDisclosureRun[] { + const runs: ToolDisclosureRun[] = []; + for (const campaign of CAMPAIGNS) { + for (const agent of AGENTS) { + runs.push( + makeRun({ + campaign, + agent, + mode: "direct", + phase: "static-visibility", + taskId: "static", + schemaTokens: 1_000, + visibleTools: 512, + }), + makeRun({ + campaign, + agent, + mode: "progressive", + phase: "static-visibility", + taskId: "static", + schemaTokens: progressiveSchemaTokens, + visibleTools: 3, + }), + ); + for (const taskId of TASK_IDS) { + runs.push( + makeRun({ + campaign, + agent, + mode: "direct", + taskId, + schemaTokens: 1_000, + latencyMs: 200, + }), + makeRun({ + campaign, + agent, + mode: "progressive", + taskId, + schemaTokens: progressiveSchemaTokens, + latencyMs: 100, + }), + ); + } + } + } + return runs; +} + +describe("tool-disclosure paired statistics", () => { + it("uses a deterministic 10,000-sample paired task bootstrap by default", () => { + const observations = [ + { direct: 10, progressive: 5 }, + { direct: 20, progressive: 16 }, + { direct: 30, progressive: 24 }, + ]; + const first = pairedBootstrapDifference(observations, { seed: 123 }); + const second = pairedBootstrapDifference(observations, { seed: 123 }); + + expect(first).toEqual(second); + expect(first.bootstrap_samples).toBe(DEFAULT_BOOTSTRAP_SAMPLES); + expect(first.paired_tasks).toBe(3); + expect(first.estimate).toBe(-5); + expect(first.lower_95).toBeLessThanOrEqual(first.estimate); + expect(first.upper_95).toBeGreaterThanOrEqual(first.estimate); + }); + + it("averages repetitions within a task before resampling task pairs", () => { + const runs = [ + makeRun({ + mode: "direct", + taskId: "task-a", + repetition: 1, + success: true, + }), + makeRun({ + mode: "direct", + taskId: "task-a", + repetition: 2, + success: false, + }), + makeRun({ + mode: "progressive", + taskId: "task-a", + repetition: 1, + success: true, + }), + makeRun({ + mode: "progressive", + taskId: "task-a", + repetition: 2, + success: true, + }), + makeRun({ mode: "direct", taskId: "task-b", success: true }), + makeRun({ mode: "progressive", taskId: "task-b", success: true }), + ]; + const [cell] = buildComparisonCells(runs, { samples: 100, seed: 1 }); + + expect(cell.differences.success_percentage_points.paired_tasks).toBe(2); + expect(cell.differences.success_percentage_points.estimate).toBe(25); + expect(cell.direct.success_rate_percent).toBe(75); + expect(cell.progressive.success_rate_percent).toBe(100); + }); +}); + +describe("tool-disclosure visibility and claim gates", () => { + it("rejects nondeterministic static visibility instead of rounding it", () => { + const runs = [ + makeRun({ + phase: "static-visibility", + mode: "direct", + schemaTokens: 1_000, + }), + makeRun({ + phase: "static-visibility", + mode: "direct", + schemaTokens: 1_001, + }), + makeRun({ + phase: "static-visibility", + mode: "progressive", + schemaTokens: 10, + }), + ]; + + expect(() => summarizeStaticVisibility(runs)).toThrow("nondeterministic"); + }); + + it("treats a success lower confidence bound of exactly -5 pp as noninferior", () => { + const runs = completeEvidence(); + const cells = buildComparisonCells(runs, { samples: 100, seed: 2 }); + const visibility = summarizeStaticVisibility(runs); + const target = cells.find( + (cell) => cell.campaign_id === "campaign-1" && cell.agent === "langchain-deepagents-code", + ); + expect(target).toBeDefined(); + const requiredTarget = target as NonNullable; + requiredTarget.differences.success_percentage_points.lower_95 = -5; + + const gates = evaluateClaimGates(cells, visibility, { + agents: ["langchain-deepagents-code"], + campaignIds: CAMPAIGNS, + primaryCatalogSize: 512, + noninferiorityMarginPercentagePoints: -5, + }); + + expect(gates.campaign_agent_gates[0].success_noninferior).toBe(true); + expect(gates.cross_agent_success_noninferior).toBe(true); + }); + + it("blocks a cross-agent claim when the improvement does not repeat in campaign two", () => { + const runs = completeEvidence(); + const cells = buildComparisonCells(runs, { samples: 100, seed: 2 }); + const visibility = summarizeStaticVisibility(runs); + const failingCell = cells.find( + (cell) => cell.campaign_id === "campaign-2" && cell.agent === "openclaw", + ); + expect(failingCell).toBeDefined(); + const requiredFailingCell = failingCell as NonNullable; + requiredFailingCell.differences.initial_tool_schema_tokens.upper_95 = 1; + + const gates = evaluateClaimGates(cells, visibility, { + agents: AGENTS, + campaignIds: CAMPAIGNS, + primaryCatalogSize: 512, + }); + + expect(gates.cross_agent_initial_schema_tokens_improved).toBe(false); + expect( + gates.campaign_agent_gates.find( + (gate) => gate.campaign_id === "campaign-2" && gate.agent === "openclaw", + )?.initial_schema_tokens_improved, + ).toBe(false); + }); + + it("does not trust a stale aggregate gate when a campaign-agent cell is blocked", () => { + const summary = buildToolDisclosureSummary(makeManifest(), completeEvidence(), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + summary.claim_gates.campaign_agent_gates[0].schema_tokens_upper_95 = 1; + + const claims = buildConservativeClaims(summary); + + expect(summary.claim_gates.cross_agent_initial_schema_tokens_improved).toBe(true); + expect(claims.some((claim) => claim.includes("initial serialized tool-schema tokens"))).toBe( + false, + ); + }); + + it("never says 99% unless every cited cell measures at least 99%", () => { + const below = buildToolDisclosureSummary(makeManifest(), completeEvidence(11), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + const atLeast = buildToolDisclosureSummary(makeManifest(), completeEvidence(10), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + const belowClaims = buildConservativeClaims(below); + const atLeastClaims = buildConservativeClaims(atLeast); + + atLeast.static_visibility[0].reduction.tokenizer_tokens_percent = 98.999; + const nearThresholdClaims = buildConservativeClaims(atLeast); + + expect(belowClaims.join(" ")).not.toContain("99%"); + expect(belowClaims[0]).toContain("initial serialized tool-schema tokens"); + expect(nearThresholdClaims.join(" ")).not.toContain("99%"); + expect(nearThresholdClaims[0]).toContain("98.9%"); + expect(atLeastClaims[0]).toContain("at least 99%"); + expect(atLeastClaims[0]).toContain("initial serialized tool-schema tokens"); + }); + + it("blocks schema and latency claims when task success is not noninferior", () => { + const runs = completeEvidence(); + for (const run of runs.filter( + (candidate) => candidate.phase === "primary" && candidate.mode === "progressive", + )) { + run.outcome = "incorrect"; + run.correctness.task_success = false; + } + const summary = buildToolDisclosureSummary(makeManifest(), runs); + expect(summary.claims.join(" ")).not.toContain("reduced initial"); + expect(summary.claims.join(" ")).not.toContain("reduced end-to-end"); + }); +}); + +describe("tool-disclosure Markdown report", () => { + it("renders only gated claims and public-safe evidence fields", () => { + const runs = completeEvidence(); + (runs[0] as ToolDisclosureRun & { prompt: string }).prompt = "RAW-PROMPT-SECRET"; + const manifest = makeManifest(); + const summary = buildToolDisclosureSummary(manifest, runs, { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + const markdown = renderToolDisclosureMarkdown(manifest, summary, { + artifacts: [ + { + artifact_id: "summary", + kind: "summary", + file_name: "summary.json", + media_type: "application/json", + byte_length: 123, + sha256: "a".repeat(64), + }, + ], + }); + + expect(markdown).toContain("# Progressive Tool-Disclosure Performance Test"); + expect(markdown).toContain("at least 99%"); + expect(markdown).toContain("initial serialized tool-schema tokens"); + expect(markdown).toContain("not an authorization boundary"); + expect(markdown).toContain("| Accelerator | 1 × example-accelerator: Example Accelerator |"); + expect(markdown).toContain("| Accelerator driver / runtime | 999.1 / example-runtime-1.0 |"); + expect(markdown).toContain("summary.json"); + expect(markdown).not.toContain("RAW-PROMPT-SECRET"); + }); + + it("renders CPU-only hardware without implying an accelerator", () => { + const manifest = makeManifest(); + manifest.environment.accelerator_type = "none"; + manifest.environment.accelerator_model = "not-applicable"; + manifest.environment.accelerator_architecture = "not-applicable"; + manifest.environment.accelerator_count = 0; + manifest.environment.accelerator_driver_version = "not-applicable"; + manifest.environment.accelerator_runtime = "not-applicable"; + const summary = buildToolDisclosureSummary(manifest, completeEvidence(), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + + const markdown = renderToolDisclosureMarkdown(manifest, summary); + + expect(markdown).toContain("| Accelerator | none |"); + expect(markdown).toContain("| Accelerator architecture | not-applicable |"); + expect(markdown).not.toContain("GPU"); + }); + + it("rejects unsafe public manifest values before rendering the report", () => { + const manifest = makeManifest(); + const summary = buildToolDisclosureSummary(manifest, completeEvidence(), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + manifest.inference.public_vllm_flags = ["--host=http://127.0.0.1:8000"]; + + expect(() => renderToolDisclosureMarkdown(manifest, summary)).toThrow(/private routing flag/u); + }); + + it("rejects artifact paths so host locations cannot enter the report", () => { + const manifest = makeManifest(); + const summary = buildToolDisclosureSummary(manifest, completeEvidence(), { + generatedAt: "2026-07-06T13:00:00.000Z", + }); + + expect(() => + renderToolDisclosureMarkdown(manifest, summary, { + artifacts: [ + { + artifact_id: "bad", + kind: "summary", + file_name: "/Users/test/summary.json", + media_type: "application/json", + byte_length: 1, + sha256: "b".repeat(64), + }, + ], + }), + ).toThrow("leaf file name"); + }); +}); diff --git a/test/performance/tool-disclosure-schedule.test.ts b/test/performance/tool-disclosure-schedule.test.ts new file mode 100644 index 0000000000..70b9a09562 --- /dev/null +++ b/test/performance/tool-disclosure-schedule.test.ts @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + appendJsonLine, + ensureCampaignDirectory, + readJsonLines, + scanArtifactsForForbiddenValues, + writeChecksumManifest, + writeJsonArtifact, +} from "../../scripts/performance/tool-disclosure/artifacts"; +import { + buildToolDisclosureSchedule, + countScheduledRunsByCampaign, +} from "../../scripts/performance/tool-disclosure/schedule"; + +const primaryTaskIds = Array.from({ length: 24 }, (_, index) => `task-${index + 1}`); +const stressTaskIds = primaryTaskIds.slice(0, 8); + +describe("tool-disclosure schedule", () => { + it("builds the frozen two-campaign matrix with adjacent balanced mode pairs", () => { + const schedule = buildToolDisclosureSchedule({ primaryTaskIds, stressTaskIds, seed: 6251 }); + expect(schedule).toHaveLength(1_884); + expect(countScheduledRunsByCampaign(schedule)).toEqual({ + "campaign-1:static-visibility": 30, + "campaign-1:small-control": 144, + "campaign-1:primary": 720, + "campaign-1:large-stress": 48, + "campaign-2:static-visibility": 30, + "campaign-2:small-control": 144, + "campaign-2:primary": 720, + "campaign-2:large-stress": 48, + }); + for (let index = 0; index < schedule.length; index += 2) { + expect(schedule[index].pair_id).toBe(schedule[index + 1].pair_id); + expect(new Set([schedule[index].mode, schedule[index + 1].mode])).toEqual( + new Set(["progressive", "direct"]), + ); + } + }); + + it("is deterministic for a fixed seed and changes order for another seed", () => { + const first = buildToolDisclosureSchedule({ primaryTaskIds, stressTaskIds, seed: 7 }); + const second = buildToolDisclosureSchedule({ primaryTaskIds, stressTaskIds, seed: 7 }); + const other = buildToolDisclosureSchedule({ primaryTaskIds, stressTaskIds, seed: 8 }); + expect(second).toEqual(first); + expect(other.map((run) => run.run_id)).not.toEqual(first.map((run) => run.run_id)); + }); +}); + +describe("tool-disclosure artifacts", () => { + it("writes JSON, JSONL, checksums, and rejects leaked values", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-tool-performance-test-")); + try { + const output = ensureCampaignDirectory(path.join(root, "campaign"), false); + writeJsonArtifact(output, "manifest.json", { safe: true }); + appendJsonLine(output, "runs.jsonl", { run_id: "one" }); + appendJsonLine(output, "runs.jsonl", { run_id: "two" }); + expect(readJsonLines(path.join(output, "runs.jsonl"))).toEqual([ + { run_id: "one" }, + { run_id: "two" }, + ]); + writeChecksumManifest(output, ["runs.jsonl", "manifest.json"]); + expect(fs.readFileSync(path.join(output, "SHA256SUMS"), "utf8")).toContain("manifest.json"); + scanArtifactsForForbiddenValues(output, ["manifest.json", "runs.jsonl"], ["secret"]); + expect(() => scanArtifactsForForbiddenValues(output, ["runs.jsonl"], ["run_id"])).toThrow( + "contains a forbidden value", + ); + expect(() => ensureCampaignDirectory(output, false)).toThrow("not empty"); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/test/performance/tool-disclosure-smoke-mcp-transport.test.ts b/test/performance/tool-disclosure-smoke-mcp-transport.test.ts new file mode 100644 index 0000000000..dd4f5723a8 --- /dev/null +++ b/test/performance/tool-disclosure-smoke-mcp-transport.test.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + PERFORMANCE_SMOKE_MCP_PORT_ENV, + PERFORMANCE_SMOKE_MCP_URL_ENV, + resolvePerformanceSmokeMcpTransport, + waitForPerformanceSmokeMcpEndpoint, +} from "../../scripts/performance/tool-disclosure/smoke-mcp-transport"; + +function externalEnv(url = "https://forwarded-smoke-123.trycloudflare.com/mcp") { + return { + [PERFORMANCE_SMOKE_MCP_URL_ENV]: url, + [PERFORMANCE_SMOKE_MCP_PORT_ENV]: "43117", + }; +} + +describe("tool-disclosure performance smoke MCP transport", () => { + it("uses the local quick tunnel when no external transport is configured", () => { + expect(resolvePerformanceSmokeMcpTransport({})).toEqual({ kind: "local-quick-tunnel" }); + }); + + it("accepts only a paired exact external tunnel URL and loopback listener port", () => { + expect(resolvePerformanceSmokeMcpTransport(externalEnv())).toEqual({ + kind: "external-ssh-forward", + mcpUrl: "https://forwarded-smoke-123.trycloudflare.com/mcp", + listenPort: 43117, + }); + expect(() => + resolvePerformanceSmokeMcpTransport({ + [PERFORMANCE_SMOKE_MCP_URL_ENV]: externalEnv()[PERFORMANCE_SMOKE_MCP_URL_ENV], + }), + ).toThrow("configured together"); + }); + + it.each([ + "http://forwarded-smoke-123.trycloudflare.com/mcp", + "https://forwarded-smoke-123.trycloudflare.com/other", + "https://forwarded-smoke-123.trycloudflare.com/mcp?token=x", + "https://user:pass@forwarded-smoke-123.trycloudflare.com/mcp", + "https://forwarded-smoke-123.trycloudflare.com.attacker.invalid/mcp", + ])("rejects unsafe external URL %s", (url) => { + expect(() => resolvePerformanceSmokeMcpTransport(externalEnv(url))).toThrow( + "exact trycloudflare /mcp URL", + ); + }); + + it.each(["0", "1023", "65536", "12x"])("rejects unsafe external listener port %s", (port) => { + expect(() => + resolvePerformanceSmokeMcpTransport({ + ...externalEnv(), + [PERFORMANCE_SMOKE_MCP_PORT_ENV]: port, + }), + ).toThrow(/port/); + }); + + it("waits through relay handoff responses until the MCP endpoint is ready", async () => { + const statuses = [502, 405]; + await expect( + waitForPerformanceSmokeMcpEndpoint("https://forwarded-smoke-123.trycloudflare.com/mcp", { + fetchImpl: (() => + Promise.resolve(new Response(null, { status: statuses.shift() ?? 405 }))) as typeof fetch, + timeoutMs: 1_000, + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/test/performance/tool-disclosure-telemetry.test.ts b/test/performance/tool-disclosure-telemetry.test.ts new file mode 100644 index 0000000000..a633226159 --- /dev/null +++ b/test/performance/tool-disclosure-telemetry.test.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + countTokensWithVllm, + parseProcessStartTime, + parseVllmTokenMetrics, + readVllmProcessStartTime, + tokenDelta, + validatedTelemetryUrl, +} from "../../scripts/performance/tool-disclosure/telemetry"; + +describe("vLLM performance test telemetry", () => { + it("sums labelled token counters and computes monotonic deltas", () => { + const snapshot = parseVllmTokenMetrics(` +# HELP vllm:prompt_tokens_total prompt +vllm:prompt_tokens_total{model_name="performance-test",engine="0"} 100 +vllm:prompt_tokens_total{model_name="performance-test",engine="1"} 20 +vllm:generation_tokens_total{model_name="performance-test",engine="0"} 30 +vllm:generation_tokens_total{model_name="performance-test",engine="1"} 2 +`); + expect(snapshot).toEqual({ prompt_tokens: 120, generation_tokens: 32 }); + expect(parseProcessStartTime("process_start_time_seconds 1712345678\n")).toBe(1712345678); + expect( + tokenDelta( + { prompt_tokens: 100, generation_tokens: 30 }, + { prompt_tokens: 120, generation_tokens: 32 }, + ), + ).toEqual({ prompt_tokens: 20, generation_tokens: 2, available: true }); + expect( + tokenDelta( + { prompt_tokens: 120, generation_tokens: 32 }, + { prompt_tokens: 100, generation_tokens: 30 }, + ).available, + ).toBe(false); + }); + + it("rejects unsafe telemetry endpoints", () => { + expect(() => validatedTelemetryUrl("http://example.com:8000", "/metrics")).toThrow( + "only on loopback", + ); + expect(() => validatedTelemetryUrl("https://user:secret@example.com", "/metrics")).toThrow( + "must not contain credentials", + ); + expect(validatedTelemetryUrl("http://127.0.0.1:8000/v1", "/metrics").toString()).toBe( + "http://127.0.0.1:8000/metrics", + ); + }); + + it("rejects non-loopback HTTPS telemetry and tokenizer URLs for the local performance test", () => { + expect(() => validatedTelemetryUrl("https://inference.example/v1", "/metrics")).toThrow( + "vLLM telemetry is allowed only on loopback", + ); + expect(validatedTelemetryUrl("https://localhost:8000/v1", "/metrics").toString()).toBe( + "https://127.0.0.1:8000/metrics", + ); + expect(validatedTelemetryUrl("https://127.0.0.1:8000/v1", "/tokenize").toString()).toBe( + "https://127.0.0.1:8000/tokenize", + ); + expect(validatedTelemetryUrl("http://127.0.0.2:8000/v1", "/metrics").toString()).toBe( + "http://127.0.0.2:8000/metrics", + ); + expect(validatedTelemetryUrl("https://[::1]:8000/v1", "/metrics").toString()).toBe( + "https://[::1]:8000/metrics", + ); + }); + + it("fetches localhost metrics and tokenizer routes through literal IPv4 loopback", async () => { + const requestedUrls: string[] = []; + const metricsFetch = (async (input: string | URL | Request) => { + requestedUrls.push(input.toString()); + return new Response("process_start_time_seconds 1712345678\n", { status: 200 }); + }) as typeof fetch; + const tokenizerFetch = (async (input: string | URL | Request) => { + requestedUrls.push(input.toString()); + return new Response(JSON.stringify({ tokens: [1, 2, 3, 4] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + + await expect(readVllmProcessStartTime("https://localhost:8443/v1", metricsFetch)).resolves.toBe( + 1712345678, + ); + await expect( + countTokensWithVllm("http://localhost:8000", "performance-test-model", "[]", tokenizerFetch), + ).resolves.toBe(4); + expect(requestedUrls).toEqual([ + "https://127.0.0.1:8443/metrics", + "http://127.0.0.1:8000/tokenize", + ]); + }); +}); diff --git a/test/performance/tool-disclosure-validation.test.ts b/test/performance/tool-disclosure-validation.test.ts new file mode 100644 index 0000000000..c4abd8fdcc --- /dev/null +++ b/test/performance/tool-disclosure-validation.test.ts @@ -0,0 +1,394 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeAll, describe, expect, it } from "vitest"; + +import { assembleToolDisclosureRun } from "../../scripts/performance/tool-disclosure/assemble-run"; +import { + canonicalJson, + generateSyntheticCatalog, + type SyntheticCatalog, + sha256Hex, +} from "../../scripts/performance/tool-disclosure/catalog"; +import type { SanitizedRunEvidence } from "../../scripts/performance/tool-disclosure/execute"; +import type { RecordedSyntheticCall } from "../../scripts/performance/tool-disclosure/grading"; +import type { ToolDisclosureRecordingEvent } from "../../scripts/performance/tool-disclosure/recorder"; +import { + buildToolDisclosureSchedule, + type ScheduledToolDisclosureRun, + STATIC_CATALOG_SIZES, + TOOL_DISCLOSURE_AGENTS, + TOOL_DISCLOSURE_MODES, +} from "../../scripts/performance/tool-disclosure/schedule"; +import { + generatePrimaryTaskSet, + generateStressTaskSet, + type SyntheticPerformanceTask, + type SyntheticTaskSet, +} from "../../scripts/performance/tool-disclosure/tasks"; +import { + DEFAULT_BOOTSTRAP_SAMPLES, + DEFAULT_BOOTSTRAP_SEED, + DEFAULT_NONINFERIORITY_MARGIN_PP, + TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + type ToolDisclosureManifest, + type ToolDisclosureRun, +} from "../../scripts/performance/tool-disclosure/types"; +import { + validateCompleteEvidence, + validateFrozenManifest, +} from "../../scripts/performance/tool-disclosure/validation"; + +interface CompleteEvidenceFixture { + catalog: SyntheticCatalog; + manifest: ToolDisclosureManifest; + primaryTasks: SyntheticTaskSet; + rawEvidence: SanitizedRunEvidence[]; + runs: ToolDisclosureRun[]; + schedule: ScheduledToolDisclosureRun[]; + stressTasks: SyntheticTaskSet; +} + +function buildManifest( + primaryTasks: SyntheticTaskSet, + stressTasks: SyntheticTaskSet, +): ToolDisclosureManifest { + const allTasks = [...primaryTasks.tasks, ...stressTasks.tasks]; + const sandboxImageDigests = Object.fromEntries( + TOOL_DISCLOSURE_AGENTS.flatMap((agent) => + TOOL_DISCLOSURE_MODES.flatMap((mode) => + STATIC_CATALOG_SIZES.map((size) => [ + `${agent}:${mode}:${size}`, + `sha256:${"a".repeat(64)}`, + ]), + ), + ), + ); + return { + schema_version: TOOL_DISCLOSURE_PERFORMANCE_TEST_SCHEMA_VERSION, + performance_test_id: "tool-disclosure-complete-evidence-fixture", + created_at: "2026-07-06T12:00:00.000Z", + sut: { + git_sha: "1".repeat(40), + git_ref: "fixture-sut", + worktree_clean: true, + }, + harness: { + git_sha: "2".repeat(40), + git_ref: "fixture-harness", + worktree_clean: true, + }, + campaigns: [ + { + campaign_id: "campaign-1", + ordinal: 1, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + { + campaign_id: "campaign-2", + ordinal: 2, + fresh_inference_process: true, + fresh_sandboxes: true, + }, + ], + protocol: { + agents: TOOL_DISCLOSURE_AGENTS, + modes: TOOL_DISCLOSURE_MODES, + catalog_sizes: STATIC_CATALOG_SIZES, + primary_catalog_size: 512, + tasks: allTasks.map((task) => ({ task_id: task.id, kind: task.kind })), + repetitions: { "small-control": 1, primary: 5, "large-stress": 1 }, + execution_seed: DEFAULT_BOOTSTRAP_SEED, + bootstrap_samples: DEFAULT_BOOTSTRAP_SAMPLES, + bootstrap_seed: DEFAULT_BOOTSTRAP_SEED, + noninferiority_margin_percentage_points: DEFAULT_NONINFERIORITY_MARGIN_PP, + retry_setup_failures: 1, + }, + environment: { + operating_system: "fixture-os", + architecture: "x86_64", + cpu_model: "fixture-cpu", + cpu_count: 1, + ram_gib: 1, + accelerator_type: "fixture-accelerator-type", + accelerator_model: "fixture-accelerator", + accelerator_architecture: "fixture-architecture", + accelerator_count: 1, + accelerator_driver_version: "fixture-driver", + accelerator_runtime: "fixture-runtime", + power_state: "fixture-power-state", + openshell_version: "fixture-openshell", + agent_versions: { + openclaw: "fixture-openclaw", + hermes: "fixture-hermes", + "langchain-deepagents-code": "fixture-deepagents", + }, + sandbox_image_digests: sandboxImageDigests, + }, + inference: { + api: "chat-completions", + model_id: "fixture-model", + model_revision: "fixture-model-revision", + container_image: "fixture-inference-image", + container_digest: `sha256:${"b".repeat(64)}`, + vllm_version: "fixture-vllm", + tool_call_parser: "fixture-tool-parser", + reasoning_parser: "fixture-reasoning-parser", + temperature: 0, + concurrency: 1, + prefix_caching_enabled: false, + public_vllm_flags: [], + }, + }; +} + +function recordedCalls(task: SyntheticPerformanceTask | undefined): RecordedSyntheticCall[] { + return ( + task?.expected_calls.map((call) => ({ + tool_name: call.tool_name, + arguments_sha256: sha256Hex(canonicalJson(call.arguments)), + result_nonce: call.result_nonce, + success: true, + })) ?? [] + ); +} + +function recordingEvent(options: { + run: ScheduledToolDisclosureRun; + toolNames: readonly string[]; +}): ToolDisclosureRecordingEvent { + return { + run_id: options.run.run_id, + request_sequence: 1, + model_call_sequence: 1, + endpoint: "chat-completions", + method: "POST", + visible_tool_count: options.toolNames.length, + canonical_tools_json_bytes: Math.max(1, options.toolNames.length * 64), + tools_sha256: sha256Hex(options.toolNames.join("\0")), + tool_names: options.toolNames, + streaming: true, + status_code: 200, + started_monotonic_ms: 1, + first_byte_monotonic_ms: 2, + ended_monotonic_ms: 3, + duration_ms: 2, + time_to_first_byte_ms: 1, + outcome: "completed", + error_reason: null, + }; +} + +function buildCompleteEvidenceFixture(): CompleteEvidenceFixture { + const catalog = generateSyntheticCatalog(); + const primaryTasks = generatePrimaryTaskSet(catalog); + const stressTasks = generateStressTaskSet(catalog); + const manifest = buildManifest(primaryTasks, stressTasks); + const schedule = buildToolDisclosureSchedule({ + primaryTaskIds: primaryTasks.tasks.map((task) => task.id), + stressTaskIds: stressTasks.tasks.map((task) => task.id), + seed: manifest.protocol.execution_seed, + }); + const tasksById = new Map( + [...primaryTasks.tasks, ...stressTasks.tasks].map((task) => [task.id, task] as const), + ); + const directToolNamesBySize: ReadonlyMap = new Map( + STATIC_CATALOG_SIZES.map((size) => [ + size, + catalog.tools.slice(0, size).map((tool) => tool.definition.function.name), + ]), + ); + const progressiveToolNames = ["search_tools"] as const; + const rawEvidence: SanitizedRunEvidence[] = []; + const runs: ToolDisclosureRun[] = []; + + for (const scheduled of schedule) { + const task = scheduled.task_id ? tasksById.get(scheduled.task_id) : undefined; + const calls = recordedCalls(task); + const toolNames = + scheduled.mode === "direct" + ? directToolNamesBySize.get(scheduled.catalog_size) + : progressiveToolNames; + expect(toolNames).toBeDefined(); + const requiredToolNames = toolNames as NonNullable; + const recorderEvents = [recordingEvent({ run: scheduled, toolNames: requiredToolNames })]; + const initialSchemaTokens = scheduled.mode === "direct" ? scheduled.catalog_size * 8 : 8; + const promptTokens = initialSchemaTokens + 100; + const completionTokens = 16; + const invocation = { exit_code: 0, timed_out: false, elapsed_ms: 10 }; + rawEvidence.push({ + run_id: scheduled.run_id, + recorder_events: recorderEvents, + calls, + invocation, + initial_schema_tokens: initialSchemaTokens, + prompt_tokens: promptTokens, + completion_tokens: completionTokens, + final_oracles_present: Boolean(task), + }); + runs.push( + assembleToolDisclosureRun({ + manifest, + scheduled, + task, + calls, + recorderEvents, + invocation: { + ...invocation, + final_output: task?.expected_final_includes.join(" ") ?? "", + }, + initialSchemaTokens, + promptTokens, + completionTokens, + }), + ); + } + + return { + catalog, + manifest, + primaryTasks, + rawEvidence, + runs, + schedule, + stressTasks, + }; +} + +describe("tool-disclosure complete-evidence validation", () => { + let fixture: CompleteEvidenceFixture; + + beforeAll(() => { + fixture = buildCompleteEvidenceFixture(); + }); + + it("rejects incomplete or mutable claim-grade manifest metadata", () => { + expect(() => validateFrozenManifest(fixture.manifest)).not.toThrow(); + + const missingModelRevision = structuredClone(fixture.manifest); + missingModelRevision.inference.model_revision = ""; + expect(() => validateFrozenManifest(missingModelRevision)).toThrow(/claim-grade/u); + + const mutableInferenceImage = structuredClone(fixture.manifest); + mutableInferenceImage.inference.container_digest = "fixture:latest"; + expect(() => validateFrozenManifest(mutableInferenceImage)).toThrow(/claim-grade/u); + }); + + it("accepts CPU-only hardware and requires explicit not-applicable accelerator metadata", () => { + const cpuOnly = structuredClone(fixture.manifest); + cpuOnly.environment.accelerator_type = "none"; + cpuOnly.environment.accelerator_model = "not-applicable"; + cpuOnly.environment.accelerator_architecture = "not-applicable"; + cpuOnly.environment.accelerator_count = 0; + cpuOnly.environment.accelerator_driver_version = "not-applicable"; + cpuOnly.environment.accelerator_runtime = "not-applicable"; + expect(() => validateFrozenManifest(cpuOnly)).not.toThrow(); + + const incompleteCpuOnly = structuredClone(cpuOnly); + incompleteCpuOnly.environment.accelerator_runtime = ""; + expect(() => validateFrozenManifest(incompleteCpuOnly)).toThrow(/claim-grade/u); + + const contradictoryCpuOnly = structuredClone(cpuOnly); + contradictoryCpuOnly.environment.accelerator_model = "fixture-accelerator"; + expect(() => validateFrozenManifest(contradictoryCpuOnly)).toThrow(/inconsistent/u); + }); + + it("accepts arbitrary accelerator metadata and rejects not-applicable positive counts", () => { + const arbitraryAccelerator = structuredClone(fixture.manifest); + arbitraryAccelerator.environment.accelerator_type = "tpu"; + arbitraryAccelerator.environment.accelerator_model = "fixture-tpu"; + arbitraryAccelerator.environment.accelerator_architecture = "fixture-tpu-architecture"; + arbitraryAccelerator.environment.accelerator_count = 4; + arbitraryAccelerator.environment.accelerator_driver_version = "fixture-tpu-driver"; + arbitraryAccelerator.environment.accelerator_runtime = "fixture-tpu-runtime"; + expect(() => validateFrozenManifest(arbitraryAccelerator)).not.toThrow(); + + arbitraryAccelerator.environment.accelerator_runtime = "not-applicable"; + expect(() => validateFrozenManifest(arbitraryAccelerator)).toThrow(/inconsistent/u); + + arbitraryAccelerator.environment.accelerator_runtime = "fixture-tpu-runtime"; + arbitraryAccelerator.environment.accelerator_type = "not-applicable"; + expect(() => validateFrozenManifest(arbitraryAccelerator)).toThrow(/inconsistent/u); + }); + + it("rejects secret-like names and values in the public manifest", () => { + for (const marker of ["api_key", "token", "secret", "password", "authorization"]) { + const secretName = structuredClone(fixture.manifest) as ToolDisclosureManifest & + Record; + secretName[marker] = "opaque-value"; + expect(() => validateFrozenManifest(secretName)).toThrow(/secret-like/u); + + const secretValue = structuredClone(fixture.manifest); + secretValue.environment.cpu_model = `${marker}=opaque-value`; + expect(() => validateFrozenManifest(secretValue)).toThrow(/secret-like/u); + } + }); + + it("rejects URL-looking values and absolute host paths in the public manifest", () => { + const endpoint = structuredClone(fixture.manifest); + endpoint.inference.model_revision = "https://example.invalid/private-revision"; + expect(() => validateFrozenManifest(endpoint)).toThrow(/URL-looking/u); + + const posixPath = structuredClone(fixture.manifest); + posixPath.environment.cpu_model = "/home/user/private-model"; + expect(() => validateFrozenManifest(posixPath)).toThrow(/absolute host path/u); + + const windowsPath = structuredClone(fixture.manifest); + windowsPath.environment.cpu_model = "C:\\Users\\user\\private-model"; + expect(() => validateFrozenManifest(windowsPath)).toThrow(/absolute host path/u); + }); + + it("rejects private routing flags and accepts reviewed public vLLM flags", () => { + for (const flag of [ + "--api-key=secret", + "--host=http://127.0.0.1:8000", + "--mount=/home/user/model", + "--port=8000", + "--enable-prefix-caching --host=0.0.0.0", + ]) { + const unsafe = structuredClone(fixture.manifest); + unsafe.inference.public_vllm_flags = [flag]; + expect(() => validateFrozenManifest(unsafe)).toThrow(/private routing flag/u); + } + + const reviewed = structuredClone(fixture.manifest); + reviewed.inference.public_vllm_flags = ["--enable-prefix-caching", "--max-model-len=8192"]; + expect(() => validateFrozenManifest(reviewed)).not.toThrow(); + }); + + it("accepts the exact 1,884-run frozen matrix and rejects reordered or tampered evidence", { + timeout: 30_000, + }, () => { + expect(fixture.schedule).toHaveLength(1_884); + expect(fixture.runs).toHaveLength(1_884); + expect(() => validateCompleteEvidence(fixture)).not.toThrow(); + + const reorderedRuns = [...fixture.runs]; + const reorderedRaw = [...fixture.rawEvidence]; + [reorderedRuns[0], reorderedRuns[1]] = [reorderedRuns[1], reorderedRuns[0]]; + [reorderedRaw[0], reorderedRaw[1]] = [reorderedRaw[1], reorderedRaw[0]]; + expect(() => + validateCompleteEvidence({ + ...fixture, + runs: reorderedRuns, + rawEvidence: reorderedRaw, + }), + ).toThrow("runs.jsonl is out of frozen schedule order"); + + const rawOnlyReordered = [...fixture.rawEvidence]; + [rawOnlyReordered[0], rawOnlyReordered[1]] = [rawOnlyReordered[1], rawOnlyReordered[0]]; + expect(() => validateCompleteEvidence({ ...fixture, rawEvidence: rawOnlyReordered })).toThrow( + "raw evidence order differs", + ); + + const tamperedRaw = [...fixture.rawEvidence]; + tamperedRaw[0] = { + ...tamperedRaw[0], + initial_schema_tokens: tamperedRaw[0].initial_schema_tokens + 1, + }; + expect(() => validateCompleteEvidence({ ...fixture, rawEvidence: tamperedRaw })).toThrow( + "does not match raw evidence", + ); + }); +}); diff --git a/test/support/dcode-start-script-fixture.ts b/test/support/dcode-start-script-fixture.ts index 9ba4e6271f..d9c9328d49 100644 --- a/test/support/dcode-start-script-fixture.ts +++ b/test/support/dcode-start-script-fixture.ts @@ -22,6 +22,7 @@ export function makeStartScriptFixture(tempDir: string): { const scriptPath = path.join(tempDir, "start.sh"); const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); + const toolDisclosureFile = path.join(tempDir, "trusted-tool-disclosure"); const original = fs.readFileSync(START_SCRIPT, "utf8"); assert.ok(original.includes("local target=/tmp/nemoclaw-proxy-env.sh")); assert.ok(original.includes('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"')); @@ -34,6 +35,10 @@ export function makeStartScriptFixture(tempDir: string): { 'readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port"', `readonly MANAGED_PROXY_PORT_FILE="${portFile}"`, ) + .replace( + 'readonly MANAGED_TOOL_DISCLOSURE_FILE="/usr/local/share/nemoclaw/dcode-tool-disclosure"', + `readonly MANAGED_TOOL_DISCLOSURE_FILE="${toolDisclosureFile}"`, + ) .replace( "readonly MANAGED_PROXY_OWNER_UID=0", `readonly MANAGED_PROXY_OWNER_UID=${process.getuid?.() ?? 0}`, @@ -49,8 +54,10 @@ export function makeStartScriptFixture(tempDir: string): { assert.ok(!fixture.includes('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"')); fs.writeFileSync(hostFile, "10.200.0.1\n", "utf8"); fs.writeFileSync(portFile, "3128\n", "utf8"); + fs.writeFileSync(toolDisclosureFile, "progressive\n", "utf8"); fs.chmodSync(hostFile, 0o444); fs.chmodSync(portFile, 0o444); + fs.chmodSync(toolDisclosureFile, 0o444); fs.writeFileSync(scriptPath, fixture, "utf8"); fs.chmodSync(scriptPath, 0o755); return { envFile, scriptPath }; diff --git a/tools/e2e/brev-remote-vitest.mts b/tools/e2e/brev-remote-vitest.mts index 19b8c30851..5ae828690d 100644 --- a/tools/e2e/brev-remote-vitest.mts +++ b/tools/e2e/brev-remote-vitest.mts @@ -8,8 +8,13 @@ export type BrevVitestProject = "cli" | "e2e-live"; export const BREV_SECURITY_SUITE_TIMEOUT_MS = 20 * 60_000; export const BREV_MESSAGING_PROVIDER_TIMEOUT_MS = 70 * 60_000; export const BREV_MESSAGING_COMPAT_TIMEOUT_MS = 40 * 60_000; +export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_TIMEOUT_MS = 55 * 60_000; export const BREV_REMOTE_WRAPPER_GRACE_MS = 120_000; export const BREV_WORKFLOW_OWNERSHIP_ENV = "NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE"; +export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE = "tool-disclosure-performance-smoke"; +export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_ARTIFACT_DIR = + "/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"; +export const BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT = 43_117; const BREV_SUITES_WITHOUT_HARNESS_SANDBOX = new Set([ "all", @@ -17,6 +22,7 @@ const BREV_SUITES_WITHOUT_HARNESS_SANDBOX = new Set([ "gpu", "messaging-compatible-endpoint", "messaging-providers", + BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_SUITE, ]); export function brevSuiteNeedsHarnessSandbox(testSuite: string): boolean { @@ -31,6 +37,47 @@ export function brevWorkflowOwnsInstance(env: NodeJS.ProcessEnv = process.env): return env[BREV_WORKFLOW_OWNERSHIP_ENV] === "1"; } +export function buildBrevSshForwardEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const selected: NodeJS.ProcessEnv = {}; + for (const name of ["PATH", "HOME", "USER", "LOGNAME", "LANG"]) { + if (env[name] !== undefined) selected[name] = env[name]; + } + for (const [name, value] of Object.entries(env)) { + if (name.startsWith("LC_") && value !== undefined) selected[name] = value; + } + return selected; +} + +export function buildBrevMcpSshForwardArgs(instanceName: string, localPort: number): string[] { + if (!instanceName) throw new Error("Brev MCP SSH forward requires an instance name"); + if (!Number.isInteger(localPort) || localPort < 1_024 || localPort > 65_535) { + throw new Error("Brev MCP SSH forward local port must be between 1024 and 65535"); + } + return [ + "-N", + "-T", + "-o", + "BatchMode=yes", + "-o", + "ForwardAgent=no", + "-o", + "StrictHostKeyChecking=no", + "-o", + "LogLevel=ERROR", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "ConnectTimeout=15", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=4", + "-L", + `127.0.0.1:${localPort}:127.0.0.1:${BREV_TOOL_DISCLOSURE_PERFORMANCE_SMOKE_MCP_PORT}`, + instanceName, + ]; +} + export function buildBrevRemoteVitestCommand(project: BrevVitestProject, target: string): string { const vitestCommand = [ "./node_modules/.bin/vitest", diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 0b3914c63a..d76f3f5f53 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -34,8 +34,8 @@ const CALLER_ALWAYS = "always()"; const MCP_SCANNED_UPLOAD_CONDITION = "${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}"; const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; -const EXPECTED_UPLOAD_JOB_COUNT = 74; -const EXPECTED_DEFAULT_CALLER_COUNT = 63; +const EXPECTED_UPLOAD_JOB_COUNT = 75; +const EXPECTED_DEFAULT_CALLER_COUNT = 64; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & {