Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
b22a972
feat(bench): add progressive disclosure proof harness
ericksoa Jul 6, 2026
4cfd7c8
merge: sync benchmark branch with main
ericksoa Jul 6, 2026
ff038cb
test(bench): keep disclosure fixtures linear
ericksoa Jul 6, 2026
8221385
fix(bench): harden benchmark evidence boundaries
ericksoa Jul 6, 2026
0ea0dd9
Merge remote-tracking branch 'origin/main' into feat/tool-disclosure-…
ericksoa Jul 6, 2026
83d1e31
fix(bench): preserve terminal failure semantics
ericksoa Jul 6, 2026
daa4224
fix(bench): enforce benchmark isolation boundaries
ericksoa Jul 6, 2026
7c18c0c
Merge remote-tracking branch 'origin/main' into feat/tool-disclosure-…
ericksoa Jul 6, 2026
dec0bdc
fix(bench): restrict vLLM traffic to loopback
ericksoa Jul 6, 2026
3e105cc
Merge remote-tracking branch 'origin/main' into feat/tool-disclosure-…
ericksoa Jul 6, 2026
252674f
refactor(bench): make disclosure protocol hardware neutral
ericksoa Jul 6, 2026
c684cf3
fix(bench): harden benchmark evidence boundaries
ericksoa Jul 6, 2026
20a64a8
fix(bench): pin localhost to literal loopback
ericksoa Jul 6, 2026
abcfec0
test: add hosted tool disclosure smoke
ericksoa Jul 7, 2026
6b23a4d
refactor: call tool disclosure a performance test
ericksoa Jul 7, 2026
8204fe1
fix(performance): preserve tool disclosure in exec shells
ericksoa Jul 7, 2026
56464c2
fix(performance): restore configured disclosure mode
ericksoa Jul 7, 2026
124b2f4
fix(performance): use service-scoped MCP token
ericksoa Jul 7, 2026
5cf30fd
fix(performance): keep agent commands single-line
ericksoa Jul 7, 2026
2ce1427
test: track the fourth cloudflared pin
ericksoa Jul 7, 2026
1fc162c
feat(performance): add compositional tool routing acceptance
ericksoa Jul 7, 2026
2ee582d
Merge remote-tracking branch 'origin/main' into feat/tool-disclosure-…
ericksoa Jul 7, 2026
61ab28b
fix(performance): honor required tool choice
ericksoa Jul 7, 2026
77af727
test(performance): keep routing coverage linear
ericksoa Jul 7, 2026
559f268
fix(performance): constrain decomposition output
ericksoa Jul 7, 2026
e203c7a
fix(performance): bound decomposer retries
ericksoa Jul 7, 2026
18e3c8e
fix(performance): harden compositional routing acceptance
ericksoa Jul 7, 2026
e0d1deb
Merge remote-tracking branch 'origin/main' into feat/tool-disclosure-…
ericksoa Jul 7, 2026
1bf2601
fix(performance): route smoke through authenticated bridge
ericksoa Jul 7, 2026
464cb99
Merge remote-tracking branch 'origin/main' into feat/tool-disclosure-…
ericksoa Jul 7, 2026
a8a2f41
fix(performance): rely on project live gate
ericksoa Jul 7, 2026
c5f9ee6
ci: run performance smoke on Brev
ericksoa Jul 7, 2026
3e07658
ci: relay Brev performance smoke tunnel
ericksoa Jul 7, 2026
c064f4a
ci: diagnose Brev tunnel readiness
ericksoa Jul 7, 2026
0c56519
ci: size Brev performance smoke host
ericksoa Jul 7, 2026
cfcf7a9
fix: support anonymous MCP snapshots on tmpfs
ericksoa Jul 7, 2026
2b90fda
fix: permit sandbox-local MCP snapshots
ericksoa Jul 7, 2026
e971120
test: keep managed snapshot probe argument safe
ericksoa Jul 7, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 95 additions & 4 deletions .github/workflows/e2e-branch-validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,17 @@ name: E2E / Branch Validation
# OpenAI-compatible endpoint. Creates its own sandbox
# (e2e-msg-compat) on a separate fresh instance.
# dashboard-remote-bind — Verifies opt-in remote dashboard forwards bind 0.0.0.0.
# tool-disclosure-performance-smoke — Runs the claim-ineligible hosted-inference
# performance smoke on an ephemeral Brev CPU VM and
# retains its sanitized evidence. (~60 min)
# gpu — Provisions a Brev GPU VM and runs the Ollama GPU E2E
# sandbox proof suite from source. (~45 min)
# all — Runs credential-sanitization + telegram-injection, each with
# its own sandbox lifecycle (NOT the independent full journey).
#
# Required secrets: BREV_API_KEY + BREV_ORG_ID (or legacy BREV_API_TOKEN), NVIDIA_INFERENCE_API_KEY
# Instance cost: Brev CPU credits (~$0.10/run for 4x16 instance)
# Instance cost: Brev CPU credits; most suites use 4x16, while the performance
# smoke requests 8 vCPU, 32 GB RAM, and 100 GB disk.

on:
workflow_dispatch:
Expand All @@ -73,6 +77,7 @@ on:
- messaging-providers
- messaging-compatible-endpoint
- dashboard-remote-bind
- tool-disclosure-performance-smoke
- gpu
- all
use_launchable:
Expand Down Expand Up @@ -188,7 +193,7 @@ jobs:
TEST_SUITE: ${{ inputs.test_suite }}
run: |
case "$TEST_SUITE" in
full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|gpu|all) ;;
full|credential-sanitization|telegram-injection|messaging-providers|messaging-compatible-endpoint|dashboard-remote-bind|tool-disclosure-performance-smoke|gpu|all) ;;
*)
echo "::error::test_suite is not one of the supported Brev E2E suites"
exit 1
Expand All @@ -213,7 +218,7 @@ jobs:
- name: Checkout target branch
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ env.RESOLVED_BRANCH || inputs.branch || 'main' }}
ref: ${{ env.RESOLVED_BRANCH || inputs.branch || github.ref }}
persist-credentials: false

- id: tested-ref
Expand Down Expand Up @@ -267,7 +272,28 @@ jobs:
- name: Build CLI
run: npm run build:cli

- name: Run ephemeral Brev E2E
- name: Install and verify Brev performance smoke tunnel prerequisite
if: inputs.test_suite == 'tool-disclosure-performance-smoke'
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"

- id: brev-test
name: Run ephemeral Brev E2E
env:
NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1"
NEMOCLAW_BREV_WORKFLOW_OWNS_INSTANCE: "1"
Expand All @@ -277,6 +303,9 @@ jobs:
INSTANCE_NAME: ${{ env.BREV_E2E_INSTANCE_NAME }}
TEST_SUITE: ${{ inputs.test_suite }}
USE_LAUNCHABLE: ${{ inputs.use_launchable && '1' || '0' }}
BREV_MIN_VCPU: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '8' || '' }}
BREV_MIN_RAM: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '32' || '' }}
BREV_MIN_DISK: ${{ inputs.test_suite == 'tool-disclosure-performance-smoke' && '100' || '' }}
BREV_PROVIDER: ${{ inputs.brev_provider || vars.BREV_PROVIDER || '' }}
BREV_GPU_TYPE: ${{ inputs.brev_gpu_type || vars.BREV_GPU_TYPE || '' }}
BREV_GPU_NAME: ${{ inputs.brev_gpu_name || vars.BREV_GPU_NAME || '' }}
Expand All @@ -286,6 +315,63 @@ jobs:
KEEP_ALIVE: ${{ inputs.keep_alive }}
run: npx vitest run --project e2e-branch-validation --silent=false --reporter=default

- name: Collect Brev tool-disclosure performance smoke artifacts
if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke'
env:
INSTANCE: ${{ env.BREV_E2E_INSTANCE_NAME }}
TEST_OUTCOME: ${{ steps.brev-test.outcome }}
run: |
set -euo pipefail
artifact_dir="brev-tool-disclosure-performance-smoke-artifacts"
remote_dir="/tmp/nemoclaw-tool-disclosure-performance-smoke-artifacts"
archive="${RUNNER_TEMP}/brev-tool-disclosure-performance-smoke-artifacts.tar.gz"
mkdir -p "$artifact_dir"
brev refresh >/dev/null
if ! ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR \
-o ConnectTimeout=10 "$INSTANCE" \
"test -d '$remote_dir' && tar -C '$remote_dir' -czf - ." \
>"$archive"; then
if [ "$TEST_OUTCOME" = "success" ]; then
echo "::error::Successful Brev performance smoke did not produce an artifact archive"
exit 1
fi
echo "::warning::Failed Brev performance smoke produced no artifact archive"
exit 0
fi
tar -C "$artifact_dir" -xzf "$archive"
result="$artifact_dir/tool-disclosure-hosted-inference-performance-smoke-completes-one-frozen-task-in-both-modes/tool-disclosure-performance-smoke.json"
if [ ! -f "$result" ]; then
if [ "$TEST_OUTCOME" = "success" ]; then
echo "::error::Successful Brev performance smoke artifact is missing its result JSON"
exit 1
fi
echo "::warning::Failed Brev performance smoke has only partial artifacts"
fi

- name: Scan Brev tool-disclosure performance smoke artifacts
if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke'
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
artifact_dir="brev-tool-disclosure-performance-smoke-artifacts"
test -d "$artifact_dir"
if [ -n "${NVIDIA_INFERENCE_API_KEY:-}" ] && \
grep -R -I -l -F -- "$NVIDIA_INFERENCE_API_KEY" "$artifact_dir" >/dev/null; then
echo "::error::Brev performance smoke artifacts contain an unredacted inference credential"
exit 1
fi

- name: Upload Brev tool-disclosure performance smoke artifacts
if: always() && inputs.test_suite == 'tool-disclosure-performance-smoke'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-brev-tool-disclosure-performance-smoke-${{ github.run_attempt }}
path: brev-tool-disclosure-performance-smoke-artifacts/
if-no-files-found: ignore
include-hidden-files: false
retention-days: 14

# Collect debugging artifacts from the Brev VM on failure before the
# instance gets torn down. Captures the onboard log, sandbox list,
# docker state, and gateway status so downstream onboard failures are
Expand All @@ -306,6 +392,11 @@ jobs:
timeout 15s openshell sandbox list > /tmp/nc-debug/sandbox-list.txt 2>&1
timeout 15s openshell gateway status > /tmp/nc-debug/gateway-status.txt 2>&1
timeout 15s docker ps -a > /tmp/nc-debug/docker-ps.txt 2>&1
free -h > /tmp/nc-debug/memory.txt 2>&1
df -h > /tmp/nc-debug/disk.txt 2>&1
timeout 15s sudo systemctl status docker --no-pager > /tmp/nc-debug/docker-service.txt 2>&1
timeout 15s sudo journalctl -u docker --since "15 minutes ago" --no-pager > /tmp/nc-debug/docker-journal.txt 2>&1
timeout 15s sudo dmesg | tail -200 > /tmp/nc-debug/kernel-tail.txt 2>&1
timeout 30s tar -C /tmp -czf /tmp/nc-debug.tar.gz nc-debug
' || true
scp -o StrictHostKeyChecking=no -o LogLevel=ERROR \
Expand Down
100 changes: 98 additions & 2 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ on:
default: ""
type: string
jobs:
description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected."
description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, mcp-bridge-dev, tool-disclosure-performance-smoke, hermes-gpu-startup, sandbox-rlimits-connect, and jetson-nvmap-gpu are skipped unless selected."
required: false
default: ""
type: string
Expand Down Expand Up @@ -723,6 +723,96 @@ jobs:
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

tool-disclosure-performance-smoke:
needs: generate-matrix
# This smoke provisions real agent sandboxes and consumes hosted inference.
# Keep it opt-in: it validates performance-test plumbing on ordinary CI hardware,
# but it is not the complete two-campaign performance-test procedure.
if: ${{ contains(format(',{0},', inputs.jobs), ',tool-disclosure-performance-smoke,') || contains(format(',{0},', inputs.targets), ',tool-disclosure-performance-smoke,') }}
runs-on: ubuntu-latest
permissions:
contents: read
timeout-minutes: 60
env:
E2E_JOB: "1"
E2E_DEFAULT_ENABLED: "0"
E2E_TARGET_ID: "tool-disclosure-performance-smoke"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/tool-disclosure-performance-smoke
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_OPENSHELL_CHANNEL: stable
NEMOCLAW_PROVIDER: custom
NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1
NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-ultra
NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-ultra
NEMOCLAW_PREFERRED_API: openai-completions
OPENSHELL_GATEWAY: nemoclaw
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- *dockerhub-auth

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28

- name: Install and verify cloudflared prerequisite
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"

- name: Install OpenShell CLI
env:
NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1"
run: bash scripts/install-openshell.sh

- name: Run tool-disclosure performance smoke live test
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx vitest run --project e2e-live \
test/e2e/live/tool-disclosure-performance-smoke.test.ts \
--silent=false --reporter=default

- name: Upload tool-disclosure performance smoke artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh

onboard-negative-paths:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.targets), ',onboard-negative-paths,') }}
Expand Down Expand Up @@ -4613,6 +4703,7 @@ jobs:
openshell-gateway-auth-contract,
mcp-bridge,
mcp-bridge-dev,
tool-disclosure-performance-smoke,
onboard-negative-paths,
skill-agent,
openclaw-skill-cli,
Expand Down Expand Up @@ -4718,6 +4809,11 @@ jobs:
target: 'mcp-bridge-dev',
reason: 'default dispatch excludes moving OpenShell dev artifacts unless explicitly selected',
},
'tool-disclosure-performance-smoke': {
job: 'tool-disclosure-performance-smoke',
target: 'tool-disclosure-performance-smoke',
reason: 'default dispatch excludes the hosted-inference performance smoke test unless explicitly selected',
},
'jetson-nvmap-gpu': {
job: 'jetson-nvmap-gpu',
target: 'jetson-nvmap-gpu',
Expand Down Expand Up @@ -4848,7 +4944,7 @@ jobs:
? '**Requested jobs:** _(selector rejected by workflow validation)_'
: requestedJobs
? `**Requested jobs:** \`${requestedJobs}\``
: '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_',
: '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `tool-disclosure-performance-smoke`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_',
`**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`,
'',
'| Job | Result |',
Expand Down
10 changes: 6 additions & 4 deletions agents/langchain-deepagents-code/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,16 @@ RUN case "$NEMOCLAW_TOOL_DISCLOSURE" in \
esac

# The launcher and startup script read these root-owned files instead of
# trusting process-level environment overrides for inference routing. Invoking
# each launcher validates the build args before the image can complete.
# trusting process-level environment overrides for inference routing or tool
# disclosure. Invoking each launcher validates the build args before the image
# can complete.
RUN install -d -m 0755 /usr/local/share/nemoclaw \
&& printf '%s\n' "$NEMOCLAW_PROXY_HOST" > /usr/local/share/nemoclaw/dcode-proxy-host \
&& printf '%s\n' "$NEMOCLAW_PROXY_PORT" > /usr/local/share/nemoclaw/dcode-proxy-port \
&& printf '%s\n' "$NEMOCLAW_TOOL_DISCLOSURE" > /usr/local/share/nemoclaw/dcode-tool-disclosure \
&& printf '%s\n' "$NEMOCLAW_INFERENCE_BASE_URL" > /usr/local/share/nemoclaw/dcode-inference-base-url \
&& chown root:root /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-inference-base-url \
&& chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-inference-base-url \
&& chown root:root /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-tool-disclosure /usr/local/share/nemoclaw/dcode-inference-base-url \
&& chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-tool-disclosure /usr/local/share/nemoclaw/dcode-inference-base-url \
&& /usr/local/bin/dcode --version \
&& /usr/local/bin/dcode.real --version \
&& /usr/local/bin/deepagents-code --version
Expand Down
2 changes: 1 addition & 1 deletion agents/langchain-deepagents-code/Dockerfile.base
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ RUN groupadd -r sandbox \

# Pre-create shell init files for the sandbox user. OpenShell/NemoClaw writes
# /tmp/nemoclaw-proxy-env.sh at startup so interactive sessions and dcode share
# the same proxy, CA, inference, HOME, and update-check posture.
# the same proxy, CA, inference, tool-disclosure, HOME, and update-check posture.
# hadolint ignore=SC2016
RUN printf '%s\n' \
'# Source runtime proxy + Deep Agents Code config' \
Expand Down
Loading
Loading