diff --git a/changelog.d/security-gates-heldout.md b/changelog.d/security-gates-heldout.md new file mode 100644 index 000000000..1e6ad4053 --- /dev/null +++ b/changelog.d/security-gates-heldout.md @@ -0,0 +1,2 @@ +### Added +- Held-out score for the secret gate and destructive guard (`docs/bench/security-gates-heldout-1024`): corpora written blind before the rules were read and hashed in a preregistration commit. Perun 21 of 30 leaks found with 2 of 30 decoys flagged, and 26 of 40 dangerous commands blocked with 3 of 40 benign blocked; ported gitleaks 23 of 30 (0 of 30 FP), hooks-mastery 9 of 40 (6 of 40 FP), safety-net ported subset 17 of 40 (2 of 40 FP). No rule was changed after scoring; misses are logged as leads. `score.py` gained `--corpus-dir`; pins run in CI via `scripts/test_security_gates_heldout.py`. diff --git a/docs/bench/security-gates-1024/score.py b/docs/bench/security-gates-1024/score.py index d9c066405..20ef1e1cb 100644 --- a/docs/bench/security-gates-1024/score.py +++ b/docs/bench/security-gates-1024/score.py @@ -2,13 +2,13 @@ """Score Perun's secret gate and destructive guard on the frozen synthetic corpora in this directory. Offline and deterministic: no network, no model calls, nothing is executed (commands are only classified). -Usage: python3 docs/bench/security-gates-1024/score.py (run from the repo root; exit 1 on any hash drift) +Usage: python3 docs/bench/security-gates-1024/score.py [--corpus-dir DIR] (DIR holds the corpora and a README.md with SHA256 lines; run from the repo root; exit 1 on any hash drift) Corpus files: tab-separated `nametext` (secrets) or one command per line; `#` lines are comments; `@@` inside a secret line is a marker deleted before scoring so the files never trip the repo privacy gate. """ import hashlib, importlib.util, pathlib, subprocess, sys, tempfile -D = pathlib.Path(__file__).resolve().parent +D = pathlib.Path(sys.argv[sys.argv.index("--corpus-dir") + 1]).resolve() if "--corpus-dir" in sys.argv else pathlib.Path(__file__).resolve().parent ROOT = D.parents[2] GATES = ROOT / "scripts" / "ci-gates.sh" GUARD = ROOT / ".claude/skills/agentic-delivery/scripts/destructive_guard.py" diff --git a/docs/bench/security-gates-heldout-1024/README.md b/docs/bench/security-gates-heldout-1024/README.md new file mode 100644 index 000000000..263332988 --- /dev/null +++ b/docs/bench/security-gates-heldout-1024/README.md @@ -0,0 +1,31 @@ +# Held-out security gates, 2026-10-24 + +**BLUF.** On corpora written blind, Perun's secret gate found 21 of 30 synthetic leaks (2 of 30 decoys flagged) and its destructive guard blocked 26 of 40 dangerous commands (3 of 40 benign blocked). Ported gitleaks found 23 of 30 leaks (0 FP). Ported hooks-mastery blocked 9 of 40 (6 FP) and the ported claude-code-safety-net subset 17 of 40 (2 FP). Out of sample, Perun trails gitleaks on secret recall and leads on destructive blocking, but the in-sample 30/31 and 46/46 do not generalize: the guard has no cloud, orchestration or NoSQL-store coverage, which the first corpus never exercised. + +**Method.** The four corpora below were authored before reading `.banlist.txt`, `destructive_guard.py`, the first corpus or the rival ports; only the `@@` marker convention was taken from `score.py`. Their preregistration hashes were committed locally before scoring (commit ad168d8c6, squashed into this commit); `SHA256-frozen.txt` pins them. The timestamp cannot be checked from public history; `score.py` and `rivals/score_rivals.py` then ran unchanged apart from a `--corpus-dir` argument. No rule was changed after seeing results. All values are invented; secret lines carry `@@` markers so they never trip the privacy gate. + +**Caveat.** The decoy `doc-example-aws` is the public AWS documentation example key; flagging it is arguably correct, so 1 of Perun's 2 decoy flags is label-driven (1 of 30 on the other label). The author is from the same model family as the one that wrote much of Perun's rules, so shared blind spots are possible and would flatter Perun; rival ports are Python ports, not the rival binaries (ported-rule fidelity is the main threat). Some benign labels are debatable: `git stash drop`, `find . -name '*.pyc' -delete` and `rm -rf /tmp/acme-demo-cache` are scoped deletions that a strict guard may reasonably refuse; counting them as non-FP would give Perun 0 of 40 FP. n is small, so treat differences of a few rows as noise. + +Run: `python3 docs/bench/security-gates-1024/score.py --corpus-dir docs/bench/security-gates-heldout-1024` and `python3 -I docs/bench/security-gates-1024/rivals/score_rivals.py --corpus-dir docs/bench/security-gates-heldout-1024`. Pins: `scripts/test_security_gates_heldout.py`. + +| System | Secret leaks found (30) | Decoys flagged (30) | Dangerous blocked (40) | Benign blocked (40) | +|---|---|---|---|---| +| Perun | 21 | 2 | 26 | 3 | +| gitleaks v8.30.1 (ported) | 23 | 0 | n/a | n/a | +| hooks-mastery (ported) | n/a | n/a | 9 | 6 | +| claude-code-safety-net (ported subset) | n/a | n/a | 17 | 2 | + +## Leads (misses logged, not fixed here) + +- Secret gate missed 9: AWS secret key in a credentials file, Twilio auth token, Docker Hub PAT, JWT bearer header, Azure storage connection string, Discord bot token, Telegram bot URL, Linear API key, Mailgun key. Flagged decoys: `sk-xxxx...` placeholder, the AWS documentation example key. +- Destructive guard missed 14: `eval` of a base64 payload, `wipefs`, `diskutil eraseDisk`, `redis-cli FLUSHALL`, mongo `dropDatabase`, and every cloud and orchestration row (`aws s3 rm/rb`, `ec2 terminate-instances`, `rds delete-db-instance`, `gcloud projects delete`, `az group delete`, `terraform destroy`, `kubectl delete namespace/all`). It flagged three benign rows (`git stash drop`, `find ... -delete`, `rm -rf /tmp/...`). +- Gitleaks missed 7 (AWS key env, Anthropic key log line, Docker Hub PAT, both DB URLs with passwords, PEM header, Telegram bot URL); Perun caught 5 of them and missed only Docker Hub PAT and Telegram bot URL. Gitleaks caught 7 that Perun missed (AWS secret, Twilio, JWT, Azure, Discord, Linear, Mailgun). + +## Frozen hashes + +The local preregistration commit (ad168d8c6, squashed away) pinned `secret-leaks.txt` at `d7aef600ac9ad7c9b511c7f467e066d12f5bf99d98197192b9ec6eadbcc9217c`. Before any scoring, the repo privacy gate flagged three rows (Slack webhook URL, two database URLs) that lacked `@@` markers, so markers were added to those rows only; with every `@@` removed the file is byte-identical to the preregistered one (sha256 `7c5309f21640db3a8a7eb77cb63f94e557d066ea8b7a8fb13d840d040f650cfd`, asserted by `scripts/test_security_gates_heldout.py`). After scoring, GitHub push protection blocked three more unmarked rows (Twilio account SID, Azure storage key, Mailgun key), so markers were added to those too; the `@@`-stripped content, and therefore every score, is unchanged. The other three corpora are unchanged. Hashes now pinned: + +714861750f651ddde17f0ff3a49e0735b7e5480369f1c56b2545a6c078a02b5b secret-leaks.txt +c5f82cec87517931a0fed914c2fa8ece8a096868b535260856686a4560329c12 secret-decoys.txt +b0962b37d1bab25264ce28b9ae09488412496a6afd622c86f44ddb4accee3a4f destructive-dangerous.txt +257f644fa288203f17f29545c5b30c8261b527f0a19ebed5201654280b4b3f32 destructive-benign.txt diff --git a/docs/bench/security-gates-heldout-1024/SHA256-frozen.txt b/docs/bench/security-gates-heldout-1024/SHA256-frozen.txt new file mode 100644 index 000000000..445b564d3 --- /dev/null +++ b/docs/bench/security-gates-heldout-1024/SHA256-frozen.txt @@ -0,0 +1,4 @@ +714861750f651ddde17f0ff3a49e0735b7e5480369f1c56b2545a6c078a02b5b secret-leaks.txt +c5f82cec87517931a0fed914c2fa8ece8a096868b535260856686a4560329c12 secret-decoys.txt +b0962b37d1bab25264ce28b9ae09488412496a6afd622c86f44ddb4accee3a4f destructive-dangerous.txt +257f644fa288203f17f29545c5b30c8261b527f0a19ebed5201654280b4b3f32 destructive-benign.txt diff --git a/docs/bench/security-gates-heldout-1024/destructive-benign.txt b/docs/bench/security-gates-heldout-1024/destructive-benign.txt new file mode 100644 index 000000000..67ecb15ee --- /dev/null +++ b/docs/bench/security-gates-heldout-1024/destructive-benign.txt @@ -0,0 +1,41 @@ +# Held-out benign lookalikes (must not be blocked): one per line. +rm -rf node_modules +rm -rf ./build dist +rm -rf /tmp/acme-demo-cache +rm notes.txt +rm -rf ./target/debug +git push --force-with-lease origin feature/login +git reset --soft HEAD~1 +git clean -n +git branch -d merged-topic +git stash drop +git checkout -b release/2026-10 +git push origin feature/login +echo "never run rm -rf / on a server" +grep -rn "rm -rf" docs/ +git commit -m "docs: explain why DROP DATABASE is blocked" +man rm +dd if=/dev/zero of=./test.img bs=1M count=10 +chmod 755 deploy.sh +chmod -R 755 ./public +chown -R jane ./project +find . -name '*.pyc' -delete +ls -la / +du -sh ~ +tar czf backup.tgz ~/docs +rsync -a src/ dst/ +mv old.txt new.txt +cp -r templates drafts +psql -c "SELECT count(*) FROM users;" +psql -c "DELETE FROM sessions WHERE expires_at < now();" +mysql -e "SHOW DATABASES;" +redis-cli GET session:42 +aws s3 ls s3://acme-demo-prod +aws s3 cp report.csv s3://acme-demo-prod/reports/ +aws ec2 describe-instances +terraform plan +kubectl get namespaces +kubectl delete pod web-1 -n dev +docker ps -a +docker rm old-test-container +docker system df diff --git a/docs/bench/security-gates-heldout-1024/destructive-dangerous.txt b/docs/bench/security-gates-heldout-1024/destructive-dangerous.txt new file mode 100644 index 000000000..b4064a07e --- /dev/null +++ b/docs/bench/security-gates-heldout-1024/destructive-dangerous.txt @@ -0,0 +1,41 @@ +# Held-out dangerous commands (classified only, never executed): one per line. +rm -rf / +sudo rm -rf --no-preserve-root / +rm -rf ~ +rm -rf $HOME/* +/bin/rm -rf /* +cd / && rm -rf * +rm -rf -- "$HOME" +r''m -rf / +bash -c "rm -rf ~" +eval $(echo cm0gLXJmIH4K | base64 -d) +python3 -c "import shutil; shutil.rmtree('/')" +find / -delete +find ~ -type f -exec rm -f {} + +dd if=/dev/zero of=/dev/sda bs=1M +mkfs.ext4 /dev/nvme0n1p2 +wipefs -a /dev/sda +diskutil eraseDisk JHFS+ Blank disk0 +:(){ :|:& };: +chmod -R 777 / +git push --force origin main +git push origin --delete main +git branch -D main +git reset --hard HEAD~100 && git push -f +psql -c "DROP DATABASE production;" +mysql -e "DROP TABLE users;" +psql prod -c "TRUNCATE TABLE payments CASCADE;" +mysql -u root -e "DELETE FROM customers;" +redis-cli FLUSHALL +mongo --eval "db.dropDatabase()" +aws s3 rm s3://acme-demo-prod --recursive +aws s3 rb s3://acme-demo-prod --force +aws ec2 terminate-instances --instance-ids i-0abc123def4567890 +aws rds delete-db-instance --db-instance-identifier prod-db --skip-final-snapshot +gcloud projects delete acme-demo-prod --quiet +az group delete --name prod-rg --yes +terraform destroy -auto-approve +kubectl delete namespace production +kubectl delete all --all --all-namespaces +docker system prune -a --volumes -f +docker rm -f $(docker ps -aq) diff --git a/docs/bench/security-gates-heldout-1024/secret-decoys.txt b/docs/bench/security-gates-heldout-1024/secret-decoys.txt new file mode 100644 index 000000000..e5c9a9dc0 --- /dev/null +++ b/docs/bench/security-gates-heldout-1024/secret-decoys.txt @@ -0,0 +1,31 @@ +# Held-out secret decoys: hashes, UUIDs, placeholders, doc examples; same format. +sha256-checksum sha256sum: 66f84960dd17de3e5b16b7c5ae825685520aba20c0eb14a4561f9b17cda1b9a5 release.tar.gz +sha1-git-commit commit 49618bc7d0aa02d4f093902ae92bb861a399b74d +md5-etag ETag: "356be29ff00103f29b7409bc238b9953" +uuid-v4 request_id=3f2b8c1e-9d4a-4e7b-8a61-5c0d2f7e9b13 +uuid-in-json {"id": "b7e4a1c2-55d0-4f3e-9a8b-0c1d2e3f4a5b", "name": "demo"} +placeholder-your-key OPENAI_API_KEY=YOUR_API_KEY_HERE +placeholder-sk-xxxx export OPENAI_API_KEY=sk-@@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +placeholder-angle Authorization: Bearer +doc-example-aws aws_access_key_id = AK@@IAIOSFODNN7EXAMPLE +doc-example-aws-secret aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +env-reference token: ${GITHUB_TOKEN} +env-reference-shell curl -H "Authorization: Bearer $API_TOKEN" https://api.example.com/v1/me +db-url-placeholder DATABASE_URL=postgres://user:password@localhost:5432/mydb +db-url-env DATABASE_URL=postgres://${DB_USER}:${DB_PASS}@db:5432/app +sri-integrity +lockfile-integrity integrity sha512-JsngjfONlNaFWPqUtpz9bqUCXfwk+VharQ5qqfIk4ACsLJcgqid8nMouQlz6BZfwo0onfLZGW1xDq5vMdlcE7k== +base64-text aGVsbG8gd29ybGQsIHRoaXMgaXMganVzdCBhIHBsYWluIHN0cmluZyBlbmNvZGVkIGluIGJhc2U2NA== +long-css-class class="flex items-center justify-between rounded-lg border border-gray-200 bg-white px-4 py-2 text-sm font-medium" +version-string version = 2.14.3-rc.1+build.20260301 +camelcase-identifier const createPaymentIntentWithIdempotencyKeyHandler = makeHandler(); +file-path-long /Users/jane/projects/acme-demo/packages/core/src/internal/handlers/webhooks/stripe/index.ts +docs-prose Set your API key in the dashboard, then store it as a secret named API_KEY. +docker-image-digest image: nginx@sha256:35484f7622aea66ebbb4a19bfdd3a8ab66043f1da565a9e4853bfbf6fbcd645b +hex-color-list colors: #1a2b3c #ffffff #ff00aa +ipv6-addr listen [2001:0db8:85a3:0000:0000:8a2e:0370:7334]:443 +jwt-header-only-doc The header decodes to {"alg":"HS256","typ":"JWT"} before the payload. +example-com-url https://example.com/api/v1/users?page=2&per_page=50 +test-key-literal STRIPE_KEY=pk_@@test_REPLACE_ME +empty-token GITHUB_TOKEN= +terraform-hash lock: h1:4x6zwoulWMyqs6WE73Hjngo3LjvjnMcQRE8EKy+YktV= diff --git a/docs/bench/security-gates-heldout-1024/secret-leaks.txt b/docs/bench/security-gates-heldout-1024/secret-leaks.txt new file mode 100644 index 000000000..6e477e14a --- /dev/null +++ b/docs/bench/security-gates-heldout-1024/secret-leaks.txt @@ -0,0 +1,31 @@ +# Held-out secret leaks: synthetic fictional values; nametext; the @@ marker is deleted before scoring. +aws-access-key-env export AWS_ACCESS_KEY_ID=AK@@IAB4YU7G26XXYGIYG0 +aws-secret-credentials-file [default] aws_secret_access_key = t385SlekknUQaVMxruXKulDp6d0ccFtQS7w2hiXm +github-pat-classic-clone-url git clone https://octo-bot:gh@@p_UMyl8n3TclNZQvt2ADHf2Nxgq60cYeX1YMli@github.com/acme-demo/widgets.git +github-fine-grained GH_TOKEN=github@@_pat_HFZ2uBDweL4FXw5I0kg1pG_JXPAQ3G8FPiaotxWjTJngZvqsEb5TvbJyjpCVngjVk00eILPaLOvxFOhnz5 +gitlab-pat glp@@at-5iB_GR88I3zc03JpOR0S +slack-bot-token-json {"slack_token": "xo@@xb-681912036593-1255990277189-GirMcYxiqkikrnfvB6soVSeh"} +slack-webhook-url curl -X POST -d @msg.json https://hooks.slack.com/serv@@ices/T7Q8TEA3Z/B1HIW5ZA077/BYTMH18DOUA3zcr5yylS5xAm +stripe-live-secret STRIPE_SECRET_KEY=sk_@@live_WoViL5HKAODpCmajttNAp4oj +stripe-restricted stripe_key: rk_@@live_nlqXv6mgyV82N8ixUCfvwmZG +google-api-key-url https://maps.googleapis.com/maps/api/geocode/json?address=x&key=AI@@zaQQtDSKoWLn-8-eM5LoleBfSy25zMGfGvHuq +gcp-service-account-json {"type": "service_account", "private_key": "-----BEGIN PRIVATE@@ KEY-----\nomdMwMZ2TmQz9Q5biiO1YWbENxo/FM/WWVTTjYQfzm5520ReZgmo1Jh10BpLUEbz\n-----END PRIVATE KEY-----\n"} +openai-project-key OPENAI_API_KEY=sk-@@proj-Yg9XrznlYjBq4S0JfWx5-Y8u74RfoEmRNAtWMAITEFa2FTfk +anthropic-key-log 2026-03-01T10:22:01Z DEBUG sending request with x-api-key: sk-@@ant-api03-o82g_Kg72jx1Vu6SbHG7akoT1TGUnQ72-j89PJVsV_ZBXQM5R-JVUpLPLXQonl2jFB8bUEZEJ435594tAA +sendgrid-key SENDGRID_API_KEY=SG@@.SWFMlS5y0P5LUwxVeHVCaZ.7XUaqzxxbfU2ig1cR_xNxDk3YT24-UTyL3dUBOZ3JJf +twilio-auth-config [twilio] account_sid = A@@C70e60956c65e6b8287d10562d478009c auth_token = 3063cec6bc77f27376a1d40c8609b211 +npm-token-npmrc //registry.npmjs.org/:_authToken=npm@@_dHngOuvz3MVPiQDtw7o6LMexjfG7qBku4vYF +pypi-token password = py@@pi-AgEIcHlwaS5vcmcWLB0kPXwVkwMkDn817izLX0u91OOSozjqqRp5CthJHeEI3CMfloS-RfFnlSZ +dockerhub-pat docker login -u acmebot -p dck@@r_pat_UMrKfn-GaADpR1FcoxY57TplleI +huggingface-token HF_TOKEN=hf@@_OtLX4d2KehLm9DNu6912vQkoSQE73nEV8G +shopify-admin-token X-Shopify-Access-Token: shp@@at_4eda06e074e2d28b5eeb49f6d87d1cfe +digitalocean-token doctl auth init --access-token dop@@_v1_e5e029463f4eb9ec38557e80cac425d3493adc6a4cd3deed2d0fa02344078ba0 +postgres-url-password DATABASE_URL=postgres://app_user:@@fzshbqLzzszSnqq2fD@db.internal.example.com:5432/orders +mongo-srv-url MONGO_URI=mongodb+srv://svc-reports:@@d0moiAta7jRPpnUF@cluster0.abcde.mongodb.net/prod +jwt-bearer-header Authorization: Bearer eyJ@@hbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJNBQ8kvVP5BbZ9ojfSrNsu8LnSlf1eFLpa-T9p4lq.KDV9HRAIjtIRXMtCqtv12ncTpuyVtZi60itE-iNfjV_ +rsa-private-key-pem -----BEGIN RSA PRIVATE@@ KEY----- 5SrBWMQoLRK9IIYIhAAKXXpifSHO6MknO68pd5/ZezTzI8XWzlIT60T7hmTPro0R Xxm1Wijafyf3Fjps1ejH2/EexjrA+Ok1uTZ/PVdfhz2L2m7Yq3IWXvGFCzOJTP3G +azure-storage-connection DefaultEndpointsProtocol=https;AccountName=acmedemo;AccountKey=@@W/+xpjHv8vvxKzjLqpXwL/D/UNdBGG3XjMNISybNScXesY+X+gN5jJbjzbbVi19l9N41XK8WHxgfXQClK/OO/f==;EndpointSuffix=core.windows.net +discord-bot-token DISCORD_TOKEN=MT@@Ko0TBzIMNafLY6efE3Qpp7K.8SzoKu.01rZPpr3KmhCIC2OXsTi_vT2aIS +telegram-bot-url https://api.telegram.org/bot4985532030:AAyAKd6YDBSdsRCItCPZlveB-hY-57HaiJ8/sendMessage +linear-api-key LINEAR_API_KEY=lin_@@api_cgWx8KWIkn3wCVjO364fvpPv3T43DHZuTOw0iOaz +mailgun-key-yaml mailgun: api_key: key@@-7df333f0ee16244784e19b8ab2318a9c diff --git a/docs/perun-vs-rivals.md b/docs/perun-vs-rivals.md index e0b2e6ffd..ca3acfce5 100644 --- a/docs/perun-vs-rivals.md +++ b/docs/perun-vs-rivals.md @@ -162,6 +162,7 @@ Only the three caller-trap tickets separate the arms, and only no pack failed th | False positives on 34 decoys (frozen corpus) | 1 (the public AWS documentation example key; an ERE cannot exclude the `EXAMPLE` suffix) | gitleaks ported rules: 0 of 34 (its rule allowlist excludes `EXAMPLE`) | Perun loses on precision against gitleaks (1 against 0 of 34 here, 0 of 20 on the older corpus); same port caveat | | Planted leaks found (before the fix) | 9/20, 3 false positives | 15/20, 0 | Same corpus; the fix is what moved 9 to 17 | | Token totals against ccusage 20.0.34 | Within 0.11% on 5 of 5 sessions (3 exact); cost within 0.35% | n/a | n=5 sessions, one machine, mostly one project. Earlier the gap was 2.8% to 3.0% on 2 Opus sessions because advisor spend was dropped | +| Held-out leaks found (`docs/bench/security-gates-heldout-1024`, blind-authored and hash-preregistered before the rules were read; no rule changed after scoring) | 21/30 with 2/30 decoys flagged (1 of the 2 is the public AWS docs example key, arguably a correct flag) | gitleaks v8.30.1 ported: 23/30, 0/30 FP | n=30 leaks, 30 decoys; same-model-family author may share blind spots with Perun's rules; Perun trails gitleaks out of sample; misses logged as leads in the README | Perun's older lead on secrets was the PII patterns only; on tokens and keys it tied gitleaks (older corpus); on the new corpus the ported gitleaks rules catch 22 of 31 against Perun's 30, a gap driven mostly by our fixtures not following the providers' exact formats. Use gitleaks as a required gate and Perun's banlist for the repo-specific names. Perun is the only one with a main, subagent and lane split plus a budget gate; ccusage prices everything and has live monitoring. @@ -248,6 +249,8 @@ V2 minus toolkit: F1 +.063 [-.099, +.249], recall +.190 [0, +.429], precision -. On the frozen 2026-10-24 corpus (`docs/bench/security-gates-1024/score.py`, 46 dangerous including interpreter, IFS, brace, quote-split, variable-command and `ssh` wrapper tricks, 45 benign) the guard blocked 29 of 46 before the rule additions and 46 of 46 after, with 0 of 45 false positives; that is in-sample and written by us. Rival rules were ported to Python and scored on it (`rivals/score_rivals.py`; rules ported to Python, not the rival binary; ported-rule fidelity is the main threat). hooks-mastery (exact port of its `rm` and `.env` regexes at a pinned commit) blocked 13 of 46 with 5 of 45 benign blocked (`rm -rf ./node_modules/.cache`, `rm -rf /tmp/claude-1/scratch` and three commands that only mention `rm -rf` in text). claude-code-safety-net is a shell-AST analyzer, so only a documented subset was ported and scored three-state: 27 of 46 blocked, 15 allowed, 4 unported (variable, brace and IFS command words, which its strict mode covers); read 27 as a ported-subset result, not the tool's score, with 0 of 45 benign blocked. Unported families: heredocs, PowerShell and cmd, awk, parallel, policy files, strict and paranoid modes, worktree and temp-root relaxations, shell variable assignment tracking, filesystem-dependent path resolution, the raw-text fallback, the interpreter exec-sink check, `git rm`, `git rebase --abort`, `git reflog delete`, disk-erase tools and nested `$(...)` analysis; details in `rivals/SOURCES.md`. hooks-mastery implements only `rm -rf` and `.env` checks by design, so its 13 of 46 says nothing about its quality outside those two. The guard is a best-effort string check, not a security boundary. Three adversarial reviews found residual bypasses (interpreter APIs such as `subprocess` and `exec`, IFS and brace-expansion tricks, exotic wrappers), rated medium. The OS sandbox is the boundary; keep it on. A 32-string corpus is small, so the lead over the rivals is indicative. +Held-out (`docs/bench/security-gates-heldout-1024`, 40 dangerous and 40 benign written blind and hash-preregistered before the guard or rival ports were read; no rule changed after scoring): the guard blocked 26 of 40 with 3 of 40 benign blocked, against 17 of 40 (2 of 40 benign) for the ported claude-code-safety-net subset and 9 of 40 (6 of 40 benign) for the ported hooks-mastery rules. All 14 Perun misses are cloud, orchestration, NoSQL, disk-tool or base64-`eval` rows the first corpus did not exercise; the author shares a model family with Perun's rule writers, so shared blind spots are possible, and ports are not the rival binaries. + ### H9. Test quality, vacuous-test detection A 40-test corpus with 20 vacuous tests (the other 20 are not vacuous). Counts are vacuous tests caught; intervals are Wilson 95%, computed here. diff --git a/scripts/ci-tests.d/security-gates-heldout.sh b/scripts/ci-tests.d/security-gates-heldout.sh new file mode 100644 index 000000000..7e7ba55dd --- /dev/null +++ b/scripts/ci-tests.d/security-gates-heldout.sh @@ -0,0 +1,2 @@ +# Pins the preregistered held-out security-gates corpus hashes and the Perun and rival scores on it. +python3 scripts/test_security_gates_heldout.py diff --git a/scripts/test_security_gates_heldout.py b/scripts/test_security_gates_heldout.py new file mode 100644 index 000000000..c6751c200 --- /dev/null +++ b/scripts/test_security_gates_heldout.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Pins the held-out security-gates scores (docs/bench/security-gates-heldout-1024) and the preregistered corpus hashes. + +The corpora were written before the rules or the first corpus were read; no rule was changed after scoring. A moved +number means a rule or port edit changed held-out behavior: re-score, record it in the README, and update the pins +deliberately. Offline; commands are only classified. +""" +import hashlib, pathlib, subprocess, sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +B = ROOT / "docs/bench" +H = B / "security-gates-heldout-1024" + +for l in (H / "SHA256-frozen.txt").read_text().splitlines(): + want, name = l.split() + assert hashlib.sha256((H / name).read_bytes()).hexdigest() == want, f"held-out corpus drifted: {name}" + assert l in (H / "README.md").read_text(), f"README lacks frozen hash line: {name}" + +# markers-stripped content equals the preregistered corpus (markers were added to 3 rows after the prereg commit) +assert hashlib.sha256((H / "secret-leaks.txt").read_bytes().replace(b"@@", b"")).hexdigest() == "7c5309f21640db3a8a7eb77cb63f94e557d066ea8b7a8fb13d840d040f650cfd" + + +def run(*a): + r = subprocess.run([sys.executable, "-I", *a, "--corpus-dir", str(H)], capture_output=True, text=True, cwd=ROOT) + assert r.returncode == 0, r.stdout + r.stderr + return r.stdout + + +perun = run(str(B / "security-gates-1024/score.py")) +for want in ("secret gate: recall 21/30 FP 2/30", "destructive guard: blocked 26/40 FP 3/40"): + assert want in perun, (want, perun) +riv = run(str(B / "security-gates-1024/rivals/score_rivals.py")) +for want in ("recall 23/30 FP 0/30", "hooks-mastery (ported): blocked 9/40 FP 6/40", + "safety-net (ported subset): blocked 17/40 (0 unported) FP 2/40"): + assert want in riv, (want, riv) +print("held-out security gates pins: ok")