bench: seeded-vulnerability security bench and scorer

changelog.d/security-seeded-bench.md

@@ -0,0 +1,2 @@
+### Added
+- Seeded-vulnerability bench for security review: `scripts/eval-fixtures/security-seeded/` holds 12 small synthetic diffs (one each of SQL injection, XSS, SSRF, path traversal, missing authorization, hardcoded secret with a fake value, unsafe deserialization, command injection, open redirect, IDOR, TOCTOU, weak hash) plus 4 look-alike safe decoys, with `keys.json` mapping each to its expected CWE and file:line. `scripts/security_bench_score.py ARM=DIR ...` scores any number of arms (for example Perun against anthropics/claude-code-security-review) on precision, recall with a bootstrap 95% CI (reusing `rival_eval.boot`) and decoy false positives, and fails closed on a missing arm dir or per-diff findings file. Offline; no model is called. Tests in `scripts/test_security_bench_score.py`, wired into `scripts/test-ci-gates.sh`.

scripts/eval-fixtures/security-seeded/diffs/01-sqli.diff

@@ -0,0 +1,11 @@
+diff --git a/app/orders.py b/app/orders.py
+new file mode 100644
+--- /dev/null
++++ b/app/orders.py
+@@ -0,0 +1,6 @@
++import sqlite3
++
++def find_orders(db, customer_name):
++    cur = db.cursor()
++    cur.execute("SELECT id, total FROM orders WHERE customer = '" + customer_name + "'")
++    return cur.fetchall()

scripts/eval-fixtures/security-seeded/diffs/02-xss.diff

@@ -0,0 +1,10 @@
+diff --git a/app/greet.py b/app/greet.py
+new file mode 100644
+--- /dev/null
++++ b/app/greet.py
+@@ -0,0 +1,5 @@
++from flask import request
++
++def greet():
++    name = request.args.get("name", "")
++    return "<h1>Hello " + name + "</h1>"

scripts/eval-fixtures/security-seeded/diffs/03-ssrf.diff

@@ -0,0 +1,12 @@
+diff --git a/app/preview.py b/app/preview.py
+new file mode 100644
+--- /dev/null
++++ b/app/preview.py
+@@ -0,0 +1,7 @@
++import urllib.request
++from flask import request
++
++def link_preview():
++    url = request.args["url"]
++    with urllib.request.urlopen(url, timeout=5) as resp:
++        return resp.read(2048)

scripts/eval-fixtures/security-seeded/diffs/04-path-traversal.diff

@@ -0,0 +1,13 @@
+diff --git a/app/files.py b/app/files.py
+new file mode 100644
+--- /dev/null
++++ b/app/files.py
+@@ -0,0 +1,8 @@
++import os
++from flask import request, send_file
++
++UPLOAD_DIR = "/srv/acme/uploads"
++
++def download():
++    name = request.args["name"]
++    return send_file(os.path.join(UPLOAD_DIR, name))

scripts/eval-fixtures/security-seeded/diffs/05-authz-bypass.diff

@@ -0,0 +1,18 @@
+diff --git a/app/admin.py b/app/admin.py
+new file mode 100644
+--- /dev/null
++++ b/app/admin.py
+@@ -0,0 +1,13 @@
++from flask import Flask, session
++
++app = Flask(__name__)
++
++@app.route("/admin/users/<int:uid>/delete", methods=["POST"])
++def delete_user(uid):
++    if "user_id" not in session:
++        return "login required", 401
++    remove_user(uid)
++    return "ok"
++
++def remove_user(uid):
++    pass

scripts/eval-fixtures/security-seeded/diffs/06-hardcoded-secret.diff

@@ -0,0 +1,11 @@
+diff --git a/app/billing.py b/app/billing.py
+new file mode 100644
+--- /dev/null
++++ b/app/billing.py
+@@ -0,0 +1,6 @@
++import requests
++
++API_KEY = "EXAMPLE_not_real_0000"
++
++def charge(amount):
++    return requests.post("https://billing.example.com/charge", auth=(API_KEY, ""), data={"amount": amount})

scripts/eval-fixtures/security-seeded/diffs/07-deserialization.diff

@@ -0,0 +1,11 @@
+diff --git a/app/session_store.py b/app/session_store.py
+new file mode 100644
+--- /dev/null
++++ b/app/session_store.py
+@@ -0,0 +1,6 @@
++import base64
++import pickle
++
++def load_session(cookie_value):
++    raw = base64.b64decode(cookie_value)
++    return pickle.loads(raw)

scripts/eval-fixtures/security-seeded/diffs/08-command-injection.diff

@@ -0,0 +1,9 @@
+diff --git a/app/thumbs.py b/app/thumbs.py
+new file mode 100644
+--- /dev/null
++++ b/app/thumbs.py
+@@ -0,0 +1,4 @@
++import subprocess
++
++def make_thumbnail(filename):
++    subprocess.run("convert " + filename + " -resize 64x64 thumb.png", shell=True, check=True)

scripts/eval-fixtures/security-seeded/diffs/09-open-redirect.diff

@@ -0,0 +1,10 @@
+diff --git a/app/login.py b/app/login.py
+new file mode 100644
+--- /dev/null
++++ b/app/login.py
+@@ -0,0 +1,5 @@
++from flask import request, redirect
++
++def after_login():
++    target = request.args.get("next", "/")
++    return redirect(target)

scripts/eval-fixtures/security-seeded/diffs/10-idor.diff

@@ -0,0 +1,14 @@
+diff --git a/app/invoices.py b/app/invoices.py
+new file mode 100644
+--- /dev/null
++++ b/app/invoices.py
+@@ -0,0 +1,9 @@
++from flask import Flask, session, jsonify
++
++app = Flask(__name__)
++
++@app.route("/invoices/<int:invoice_id>")
++def get_invoice(invoice_id):
++    if "user_id" not in session:
++        return "login required", 401
++    return jsonify(Invoice.query.get(invoice_id).to_dict())

scripts/eval-fixtures/security-seeded/diffs/11-toctou.diff

@@ -0,0 +1,11 @@
+diff --git a/app/export.py b/app/export.py
+new file mode 100644
+--- /dev/null
++++ b/app/export.py
+@@ -0,0 +1,6 @@
++import os
++
++def write_report(path, data):
++    if not os.path.exists(path):
++        with open(path, "w") as fh:
++            fh.write(data)

scripts/eval-fixtures/security-seeded/diffs/12-crypto-misuse.diff

@@ -0,0 +1,9 @@
+diff --git a/app/passwords.py b/app/passwords.py
+new file mode 100644
+--- /dev/null
++++ b/app/passwords.py
+@@ -0,0 +1,4 @@
++import hashlib
++
++def hash_password(password):
++    return hashlib.md5(password.encode()).hexdigest()

scripts/eval-fixtures/security-seeded/diffs/13-decoy-sqli-param.diff

@@ -0,0 +1,9 @@
+diff --git a/app/orders_safe.py b/app/orders_safe.py
+new file mode 100644
+--- /dev/null
++++ b/app/orders_safe.py
+@@ -0,0 +1,4 @@
++def find_orders(db, customer_name):
++    cur = db.cursor()
++    cur.execute("SELECT id, total FROM orders WHERE customer = ?", (customer_name,))
++    return cur.fetchall()

scripts/eval-fixtures/security-seeded/diffs/14-decoy-escaped-html.diff

@@ -0,0 +1,11 @@
+diff --git a/app/greet_safe.py b/app/greet_safe.py
+new file mode 100644
+--- /dev/null
++++ b/app/greet_safe.py
+@@ -0,0 +1,6 @@
++from flask import request
++from markupsafe import escape
++
++def greet():
++    name = request.args.get("name", "")
++    return "<h1>Hello " + str(escape(name)) + "</h1>"

scripts/eval-fixtures/security-seeded/diffs/15-decoy-fixed-argv.diff

@@ -0,0 +1,9 @@
+diff --git a/app/thumbs_safe.py b/app/thumbs_safe.py
+new file mode 100644
+--- /dev/null
++++ b/app/thumbs_safe.py
+@@ -0,0 +1,4 @@
++import subprocess
++
++def make_thumbnail(path):
++    subprocess.run(["convert", "--", path, "-resize", "64x64", "thumb.png"], check=True)

scripts/eval-fixtures/security-seeded/diffs/16-decoy-env-secret.diff

@@ -0,0 +1,13 @@
+diff --git a/app/billing_safe.py b/app/billing_safe.py
+new file mode 100644
+--- /dev/null
++++ b/app/billing_safe.py
+@@ -0,0 +1,8 @@
++import os
++import requests
++
++API_KEY = os.environ["BILLING_API_KEY"]
++EXAMPLE_PLACEHOLDER = "replace-me-in-your-env-file"
++
++def charge(amount):
++    return requests.post("https://billing.example.com/charge", auth=(API_KEY, ""), data={"amount": amount})

scripts/eval-fixtures/security-seeded/keys.json

@@ -0,0 +1,118 @@
+{
+  "01-sqli": {
+    "cwe": "CWE-89",
+    "file": "app/orders.py",
+    "line": 5,
+    "marker": "execute(",
+    "equiv": []
+  },
+  "02-xss": {
+    "cwe": "CWE-79",
+    "file": "app/greet.py",
+    "line": 5,
+    "marker": "\"<h1>Hello \"",
+    "equiv": []
+  },
+  "03-ssrf": {
+    "cwe": "CWE-918",
+    "file": "app/preview.py",
+    "line": 6,
+    "marker": "urlopen(",
+    "equiv": []
+  },
+  "04-path-traversal": {
+    "cwe": "CWE-22",
+    "file": "app/files.py",
+    "line": 8,
+    "marker": "send_file(",
+    "equiv": [
+      "CWE-23",
+      "CWE-73"
+    ]
+  },
+  "05-authz-bypass": {
+    "cwe": "CWE-862",
+    "file": "app/admin.py",
+    "line": 9,
+    "marker": "remove_user(uid)",
+    "equiv": [
+      "CWE-306",
+      "CWE-285",
+      "CWE-284"
+    ]
+  },
+  "06-hardcoded-secret": {
+    "cwe": "CWE-798",
+    "file": "app/billing.py",
+    "line": 3,
+    "marker": "API_KEY =",
+    "equiv": [
+      "CWE-259",
+      "CWE-321"
+    ]
+  },
+  "07-deserialization": {
+    "cwe": "CWE-502",
+    "file": "app/session_store.py",
+    "line": 6,
+    "marker": "pickle.loads",
+    "equiv": []
+  },
+  "08-command-injection": {
+    "cwe": "CWE-78",
+    "file": "app/thumbs.py",
+    "line": 4,
+    "marker": "shell=True",
+    "equiv": [
+      "CWE-77"
+    ]
+  },
+  "09-open-redirect": {
+    "cwe": "CWE-601",
+    "file": "app/login.py",
+    "line": 5,
+    "marker": "redirect(",
+    "equiv": []
+  },
+  "10-idor": {
+    "cwe": "CWE-639",
+    "file": "app/invoices.py",
+    "line": 9,
+    "marker": "Invoice.query.get",
+    "equiv": [
+      "CWE-284",
+      "CWE-863"
+    ]
+  },
+  "11-toctou": {
+    "cwe": "CWE-367",
+    "file": "app/export.py",
+    "line": 5,
+    "marker": "open(path",
+    "equiv": [
+      "CWE-362"
+    ]
+  },
+  "12-crypto-misuse": {
+    "cwe": "CWE-327",
+    "file": "app/passwords.py",
+    "line": 4,
+    "marker": "md5(",
+    "equiv": [
+      "CWE-328",
+      "CWE-916"
+    ]
+  },
+  "13-decoy-sqli-param": {
+    "cwe": null
+  },
+  "14-decoy-escaped-html": {
+    "cwe": null
+  },
+  "15-decoy-fixed-argv": {
+    "cwe": null
+  },
+  "16-decoy-env-secret": {
+    "cwe": null
+  }
+}