Close the generalizable held-out security-gate misses and score them out of sample

.banlist.txt

@@ -35,6 +35,16 @@ pypi-AgE[A-Za-z0-9_-]{50,}
 (^|[^A-Za-z0-9])hf_[A-Za-z0-9]{34,}
 (^|[^A-Za-z0-9])shp(at|ca|pa|ss)_[a-f0-9]{32}
 (^|[^A-Za-z0-9])do[por]_v1_[a-f0-9]{64}
+(aws_secret_access_key|AWS_SECRET_ACCESS_KEY)[ ]*[=:][ ]*["']?[A-Za-z0-9/+]{30}([A-DF-Za-z0-9/+][A-Za-z0-9/+]{9}|E[A-WY-Za-z0-9/+][A-Za-z0-9/+]{8}|EX[B-Za-z0-9/+][A-Za-z0-9/+]{7}|EXA[A-LN-Za-z0-9/+][A-Za-z0-9/+]{6}|EXAM[A-OQ-Za-z0-9/+][A-Za-z0-9/+]{5}|EXAMP[A-KM-Za-z0-9/+][A-Za-z0-9/+]{4}|EXAMPL[A-DF-Za-z0-9/+][A-Za-z0-9/+]{3}|EXAMPLE[A-JL-Za-z0-9/+][A-Za-z0-9/+]{2}|EXAMPLEK[A-DF-Za-z0-9/+][A-Za-z0-9/+]{1}|EXAMPLEKE[A-XZ-Za-z0-9/+])([^A-Za-z0-9/+]|$)
+(^|[^A-Za-z0-9])AC[0-9a-f]{32}([^A-Za-z0-9]|$)
+(auth_token|AUTH_TOKEN|authToken)["']?[ ]*[:=][ ]*["']?[0-9a-f]{32}([^0-9A-Za-z]|$)
+(^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}
+(AccountKey|SharedAccessKey)=[A-Za-z0-9+/]{40,}
+(^|[^A-Za-z0-9_-])[MNO][A-Za-z0-9_-]{23,27}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,38}([^A-Za-z0-9_-]|$)
+lin_(api|oauth)_[A-Za-z0-9]{40}
+(^|[^A-Za-z0-9-])key-[0-9a-f]{32}([^0-9A-Za-z]|$)
+dckr_pat_[A-Za-z0-9_-]{20,}
+(^|[^0-9])[0-9]{8,10}:AA[A-Za-z0-9_-]{33}([^A-Za-z0-9_-]|$)
 [A-Za-z][A-Za-z0-9+.-]*://[A-Za-z0-9._-]+:[A-Za-z0-9%+_=-]{12,}@[A-Za-z0-9-]+\.[A-Za-z]
 -----BEGIN (RSA |EC |OPENSSH |DSA |ENCRYPTED )?PRIVATE KEY-----
 -----BEGIN (PGP )PRIVATE KEY BLOCK-----

.claude/skills/agentic-delivery/scripts/destructive_guard.py

@@ -44,7 +44,10 @@
   `docker volume rm|prune`, `docker system prune --volumes|-a`, `docker rm|rmi|kill $(...)` (global options like `-H x` are skipped), `kill ... -1|0`, `shutdown|reboot|halt|poweroff`, `shred`, `truncate -s 0`,
   `rsync --delete` (not with `-n`/`--dry-run`), SQL DROP/TRUNCATE/DELETE-all via psql/mysql/sqlite3, fork bombs,
   `git push --force-with-lease` to main/master, `git worktree remove --force`, a secret variable in a curl/wget
-  URL query.
+  URL query, `aws s3 rm --recursive|s3 rb --force|ec2 terminate-instances|rds delete-db-*`, `gcloud projects delete`,
+  `az group delete`, `terraform|tofu destroy|apply -destroy`, `kubectl delete namespace|all --all`,
+  `redis-cli FLUSHALL|FLUSHDB`, mongo `dropDatabase`, `wipefs` (not `-n`), `diskutil eraseDisk|eraseVolume|...`,
+  and `eval`/`sh` of a `$(... base64 -d ...)` payload.
 Everything else is allowed.
 
 No in-repo grant or allow list: anything an agent can write, an agent can forge. When the owner really wants a
@@ -86,6 +89,9 @@
 DYNCMD = re.compile(r"(?:^|[;&|\n({]|\bthen\b|\bdo\b)\s*\"?(?:\$\([^()]{0,200}\)|`[^`]{0,200}`|\$\{?\w{1,64}\}?)\"?(?:\s+\S+){0,8}?\s+(?:-\w*(?:rf|fr)\w*|--hard)\b")
 FORKBOMB = re.compile(r"(\S{1,32})\(\)\s*\{\s*\1\s*\|\s*\1\s*&")
 SSH_VAL = "bcDEeFIiJLlmOopQRSWw"
+B64EXEC = re.compile(r"\b(?:eval|source|(?:ba|z|da|k|a)?sh)\b[^|;&\n]{0,300}(?:\$\(|`)[^)`]{0,300}\bbase64\b[^)`]{0,300}(?:\s-\w*[dD]|--decode)")
+KUBE_VAL = ("-n", "--namespace", "-l", "--selector", "-f", "--filename", "--context", "--cluster", "--kubeconfig", "--user", "-o", "--output", "--field-selector")
+DISKUTIL = ("erasedisk", "erasevolume", "secureerase", "zerodisk", "randomdisk", "partitiondisk")
 DOCKER_VAL = ("-H", "--host", "--context", "-c", "--config", "-l", "--log-level", "--tlscacert", "--tlscert", "--tlskey")
 HEREDOC = re.compile(r"<<-?[ \t]*(['\"]?)([A-Za-z_]\w*)\1[^\n]*\n(.*?)\n[ \t]*\2[ \t]*(?=\n|$)", re.S)
 MAXDEPTH = 6
@@ -95,6 +101,8 @@
     r"\bgit\b.*\b(?:reset\s+--ha|clean\s+-\w*f|checkout\s+(?:-f|--\s+\.|\.)|switch\s+-f|restore\s+\.|branch\s+-D|stash\s+(?:clear|drop))",
     r"\bgit\b.*\bpush\b.*(?:--for|\s-\w*f\w*\b|--del|--mir|\s[+:]\S)",
     r"\bfind\b.*(?:-delete|-exec\s+rm)", r"\bdd\b.*\bof=/dev/(?!null|zero|std)", r"\bmkfs",
+    r"\b(?:terraform|tofu)\b.*\bdestroy\b", r"\bkubectl\b.*\bdelete\s+(?:ns|namespaces?)\b",
+    r"\baws\b.*\b(?:terminate-instances|delete-db-|s3\s+rb)", r"\b(?:wipefs|redis-cli\s.*flush(?:all|db))\b",
     r"\b(?:chmod|chown)\s+(?:-\S+\s+)*-\w*R\w*\s+(?:\S+\s+)?[/~]", r"\b(?:pkill|killall)\b",
     r"\bkill\s+(?!(?:-0|-s\s+0)(?:\s+--)?\s+(?:\d+|\"?\$\{?\w+\}?\"?|\"\$\([^\"()]*\)\")\s*(?:[;&|\n]|$)).*[$`]", r"\b(?:curl|wget)\b.*\|\s*(?:sudo\s+)?(?:ba|z|da|k|fi)?sh\b",
 )]
@@ -242,6 +250,33 @@ def _cmd(name, a, ph):
         return "kill by pattern/substitution. `kill` the exact PID you started"
     if name in ("curl", "wget") and any(re.match(r"https?://\S*\?\S*\$\{?\w*(?:SECRET|TOKEN|PASSW|API_?KEY|PRIVATE)", t, re.I) for t in a):
         return "secret variable in a URL query string leaves the host. Send it in an Authorization header to a host you trust"
+    w = [t for t in a if not t.startswith("-")]  # positional words; option values may leak in, which only widens a match
+    pair = lambda *x: any(w[k:k + len(x)] == list(x) for k in range(len(w)))
+    if (name == "aws" and ((pair("s3", "rm") and "--recursive" in a) or (pair("s3", "rb") and "--force" in a)
+                           or pair("ec2", "terminate-instances") or any(t.startswith("delete-db-") for t in w) and "rds" in w)) \
+            or (name == "gcloud" and pair("projects", "delete")) or (name == "az" and pair("group", "delete")):
+        return f"{name} bulk or whole-resource delete. Name the exact resource and ask the owner"
+    if name in ("terraform", "tofu") and (w[:1] == ["destroy"] or (w[:1] == ["apply"] and any(t in ("-destroy", "--destroy") for t in a))):
+        return f"{name} destroy removes real infrastructure. Run `plan -destroy` and ask the owner"
+    if name == "kubectl" and "delete" in w:
+        r, k = [], a.index("delete") + 1 if "delete" in a else len(a)  # first positional after delete, skipping option values (`-n foo ns bar`)
+        while k < len(a) and not r:
+            if a[k] in KUBE_VAL:
+                k += 1
+            elif not a[k].startswith("-"):
+                r = [a[k]]
+            k += 1
+        r = r[0].split("/")[0].lower() if r else ""
+        if r in ("ns", "namespace", "namespaces") or (r == "all" and ("--all" in a or "-A" in a or "--all-namespaces" in a)):
+            return "kubectl delete namespace/all removes a whole workload. Delete exact named objects and ask the owner"
+    if name == "redis-cli" and any(t.upper() in ("FLUSHALL", "FLUSHDB") for t in a):
+        return "redis FLUSHALL/FLUSHDB wipes the keyspace. Ask the owner"
+    if name in ("mongo", "mongosh") and any(re.search(r"dropDatabase\s*\(", t) for t in a):
+        return "mongo dropDatabase. Ask the owner"
+    if name == "wipefs" and not any(t in ("-n", "--no-act") for t in a):
+        return "wipefs erases filesystem signatures on a device. Do it in a VM, never on the host"
+    if name == "diskutil" and w[:1] and w[0].lower() in DISKUTIL:
+        return "diskutil erase/partition wipes a disk. Do it in a VM, never on the host"
     if name == "ssh":  # the remote command is everything after the host
         k = 0
         while k < len(a) and a[k].startswith("-"):
@@ -353,12 +388,16 @@ def _feed(name, text):
 def _bodies(cmd):
     """Backtick and `$(...)` bodies of the raw text (balanced parens), scanned recursively by the caller."""
     out = re.findall(r"`([^`]*)`", cmd)
+    end = 0
     for m in re.finditer(r"\$\(", cmd):
+        if m.start() < end:
+            continue  # nested: the outer body already holds it and check() recurses into it (no fan-out blowup)
         depth, j = 1, m.end()
         while j < len(cmd) and depth:
             depth += (cmd[j] == "(") - (cmd[j] == ")")
             j += 1
         out.append(cmd[m.end():j - 1 if depth == 0 else j])
+        end = j
     return out
 
 
@@ -383,6 +422,8 @@ def check(cmd, _depth=0, _ph=None):
     cmd = re.sub(r"\$\{IFS\}|\$IFS\b", " ", cmd)  # rm${IFS}-rf is rm -rf
     if "()" in cmd and FORKBOMB.search(cmd):
         return "fork bomb. Never run it"
+    if "base64" in cmd and B64EXEC.search(cmd):
+        return "eval/shell of a decoded base64 payload cannot be vetted. Decode to a file, read it, then run it"
     if DYNCMD.search(cmd):
         return "command word built from a variable or substitution with -rf/--hard. Spell the command out"
     try:
@@ -461,7 +502,7 @@ def check(cmd, _depth=0, _ph=None):
         if r:
             return r
         src = src or hd[n] or name in ("echo", "printf", "cat", "tee") or (
-            name == "base64" and any(re.fullmatch(r"-[A-Za-z]*d[A-Za-z]*|--decode", t) for t in a))
+            name == "base64" and any(re.fullmatch(r"-[A-Za-z]*[dD][A-Za-z]*|--decode", t) for t in a))
         fetch = fetch or name in ("curl", "wget")
         feed = ([" ".join(a)] + a) if name in ("echo", "printf") else None  # joined, and each arg (printf FORMAT ARG)
         prev = name

changelog.d/security-gates-heldout-fixes.md

@@ -0,0 +1,5 @@
+### Added
+- `.banlist.txt` provider-format patterns for the AWS secret access key in a credentials file (the public AWS docs example secret stays allowed), Twilio account SID and auth token, JWT, Azure storage `AccountKey`/`SharedAccessKey`, Discord bot token, Linear `lin_api_`/`lin_oauth_`, Mailgun `key-`, Docker Hub `dckr_pat_` and Telegram bot token.
+- `destructive_guard.py` now blocks `aws s3 rm --recursive`, `aws s3 rb --force`, `ec2 terminate-instances`, `rds delete-db-*`, `gcloud projects delete`, `az group delete`, `terraform|tofu destroy` and `apply -destroy`, `kubectl delete namespace` and `delete all --all`, `redis-cli FLUSHALL|FLUSHDB`, mongo `dropDatabase`, `wipefs`, `diskutil eraseDisk` and siblings, and `eval`/`sh` of a decoded base64 payload. Routine `kubectl delete pod`, `terraform plan` and a single-key `aws s3 rm` stay allowed.
+- Held-out-1024 after the fixes: secret gate 21/30 to 30/30 (2/30 decoys flagged, unchanged), guard 26/40 to 40/40 (3/40 benign, unchanged); the first corpus stays 30/31 and 46/46 with FP 1/34 and 0/45. The held-out corpus is now in-sample; a fresh held-out-2 corpus is the real test. Tests: `scripts/test_security_gates_heldout_fixes.py`.
+- Out-of-sample score (`docs/bench/security-gates-heldout2-1024`, preregistered corpus, rules frozen at this change): secret gate 23/40 leaks with 2/40 decoys flagged against ported gitleaks 26/40 (0 FP); guard 37/50 with 4/50 benign blocked against ported safety-net 18/50 (2 FP) and hooks-mastery 9/50 (5 FP). Perun still trails gitleaks on secrets out of sample. the privacy gate skips the frozen `security-gates-*/secret-leaks.txt` and `secret-decoys.txt` corpora (fictional values, immutable once landed); `score.py` accepts `id<TAB>cmd` rows and hashes from `SHA256-frozen.txt` (last line per path); `bench_check.py` resolves a bare frozen filename from the bench dir first. Pins: `scripts/test_security_gates_heldout2.py`.

changelog.d/security-gates-heldout.md

@@ -1,2 +1,2 @@
 ### Added
-- Held-out score for the secret gate and destructive guard (`docs/bench/security-gates-heldout-1024`): corpora written blind before the rules were read and hashed in a preregistration commit. Perun 21 of 30 leaks found with 2 of 30 decoys flagged, and 26 of 40 dangerous commands blocked with 3 of 40 benign blocked; ported gitleaks 23 of 30 (0 of 30 FP), hooks-mastery 9 of 40 (6 of 40 FP), safety-net ported subset 17 of 40 (2 of 40 FP). No rule was changed after scoring; misses are logged as leads. `score.py` gained `--corpus-dir`; pins run in CI via `scripts/test_security_gates_heldout.py`.
+- Held-out score for the secret gate and destructive guard (`docs/bench/security-gates-heldout-1024`): corpora written blind before the rules were read and hashed in a preregistration commit. Perun 21 of 30 leaks found with 2 of 30 decoys flagged, and 26 of 40 dangerous commands blocked with 3 of 40 benign blocked; ported gitleaks 23 of 30 (0 of 30 FP), hooks-mastery 9 of 40 (6 of 40 FP), safety-net ported subset 17 of 40 (2 of 40 FP). No rule was changed during scoring; misses were logged as leads and closed afterwards (see the gate-fix entry). `score.py` gained `--corpus-dir`; pins run in CI via `scripts/test_security_gates_heldout.py`.

docs/INDEX.md

@@ -15,7 +15,7 @@ Generated by `python3 .claude/skills/agentic-delivery/scripts/focus_kit.py docs-
 | `docs/idea-critic-outcome-1019/README.md` | idea-critic outcome bench (2026-10-19), exploratory (not preregistered): on 36 historically resolved go/no-go decisions, the idea-critic sk… | 2085 |
 | `docs/peer-survey-2026-10-16.md` | Peer survey and field measurements, 2026-10-16: The roadmap in roadmap.md cites this page as the source for peer-reported pain points. | 335 |
 | `docs/perun-doctor.md` | Perun doctor and uninstall: Installed scripts and hooks can sit in a repo and never run: an operating layer that needed an extra flag, a st… | 1588 |
-| `docs/perun-vs-rivals.md` | Perun vs other skill bundles: Harness: run rival rows with scripts/rival_eval_keys.py run ..., a wrapper around the hash-pinned scripts/riv… | 31990 |
+| `docs/perun-vs-rivals.md` | Perun vs other skill bundles: Harness: run rival rows with scripts/rival_eval_keys.py run ..., a wrapper around the hash-pinned scripts/riv… | 32305 |
 | `docs/review-cost-per-recall.md` | Review cost per real bug caught, and false alarms per case: on the one set with per-case data (S1, n=22, patch-only), Perun's real skill ca… | 2915 |
 | `docs/roadmap.md` | Roadmap: Two parts. The product roadmap below is what to build or fix next for Perun, produced by the CEO improvement loop: ranked by a pri… | 15600 |
 | `docs/scoreboard.md` | Perun scoreboard: Generated by scripts/scoreboard.py from committed results | 4422 |

docs/bench/security-gates-1024/rivals/score_rivals.py

@@ -288,7 +288,7 @@ def scores():
     if GL[2]: raise SystemExit(f"{GL[2]} gitleaks regexes failed to compile")
     pick = lambda n: [(l.split("\t")[0], l.split("\t", 1)[1].replace("@@", "")) for l in sc.rows(n)]
     leaks, decoys = pick("secret-leaks.txt"), pick("secret-decoys.txt")
-    dang, ben = sc.rows("destructive-dangerous.txt"), sc.rows("destructive-benign.txt")
+    dang, ben = sc.cmds("destructive-dangerous.txt"), sc.cmds("destructive-benign.txt")
     s = {"gitleaks": dict(n_leaks=len(leaks), n_decoys=len(decoys), rules=len(GL[1]),
                           missed=[n for n, t in leaks if not gitleaks_rule(t)], fp=[n for n, t in decoys if gitleaks_rule(t)])}
     s["hooks-mastery"] = dict(n=len(dang), n_ben=len(ben), missed=[c for c in dang if not hooks_mastery(c)], fp=[c for c in ben if hooks_mastery(c)])

docs/bench/security-gates-1024/score.py

@@ -2,8 +2,8 @@
 """Score Perun's secret gate and destructive guard on the frozen synthetic corpora in this directory.
 
 Offline and deterministic: no network, no model calls, nothing is executed (commands are only classified).
-Usage: python3 docs/bench/security-gates-1024/score.py [--corpus-dir DIR]   (DIR holds the corpora and a README.md with SHA256 lines; run from the repo root; exit 1 on any hash drift)
-Corpus files: tab-separated `name<TAB>text` (secrets) or one command per line; `#` lines are comments; `@@`
+Usage: python3 docs/bench/security-gates-1024/score.py [--corpus-dir DIR]   (DIR holds the corpora and SHA256 lines in SHA256-frozen.txt, else README.md; run from the repo root; exit 1 on any hash drift)
+Corpus files: tab-separated `name<TAB>text` (secrets) or `id<TAB>cmd` / one command per line; `#` lines are comments; `@@`
 inside a secret line is a marker deleted before scoring so the files never trip the repo privacy gate.
 """
 import hashlib, importlib.util, pathlib, subprocess, sys, tempfile
@@ -18,13 +18,19 @@ def rows(name):
     return [l for l in (D / name).read_text().splitlines() if l and not l.startswith("#")]
 
 
+def cmds(name):
+    """Command rows: `id<TAB>cmd` (the id is dropped) or a bare command per line."""
+    return [l.split("\t", 1)[-1] for l in rows(name)]
+
+
 def sha_ok():
-    """Compare corpus hashes with SHA256 lines in README.md; returns the list of drifted files."""
+    """Compare corpus hashes with the SHA256 lines of SHA256-frozen.txt (last line per path pins the file, earlier ones are disclosed amendments) or, without it, README.md; returns the list of drifted files."""
     want = {}
-    for l in (D / "README.md").read_text().splitlines():
+    fz = D / "SHA256-frozen.txt"
+    for l in (fz if fz.exists() else D / "README.md").read_text().splitlines():
         p = l.split()
-        if len(p) == 2 and len(p[0]) == 64 and p[1].endswith(".txt"):
-            want[p[1]] = p[0]
+        if len(p) == 2 and len(p[0]) == 64 and p[1].endswith((".txt", ".md") if fz.exists() else ".txt"):
+            want[pathlib.PurePath(p[1]).name] = p[0]
     return [f for f, h in want.items() if hashlib.sha256((D / f).read_bytes()).hexdigest() != h] or (
         [] if want else ["README.md has no sha256 lines"])
 
@@ -51,7 +57,7 @@ def main():
         fp = [n for n, t in decoys if secret_flagged(tmp, t)]
     print(f"secret gate: recall {len(leaks) - len(miss)}/{len(leaks)}  FP {len(fp)}/{len(decoys)}")
     print("  missed:", miss); print("  false positives:", fp)
-    dang, ben = rows("destructive-dangerous.txt"), rows("destructive-benign.txt")
+    dang, ben = cmds("destructive-dangerous.txt"), cmds("destructive-benign.txt")
     dmiss = [c for c in dang if not dg.check(c)]
     dfp = [c for c in ben if dg.check(c)]
     print(f"destructive guard: blocked {len(dang) - len(dmiss)}/{len(dang)}  FP {len(dfp)}/{len(ben)}")