diff --git a/.github/workflows/retrieval-sidecar-smoke.yml b/.github/workflows/retrieval-sidecar-smoke.yml index a3d4175d..98a47902 100644 --- a/.github/workflows/retrieval-sidecar-smoke.yml +++ b/.github/workflows/retrieval-sidecar-smoke.yml @@ -6,66 +6,46 @@ name: retrieval-sidecar-smoke on: pull_request: paths: - - crates/codestory-retrieval/** - - crates/codestory-contracts/** - - crates/codestory-store/Cargo.toml - - crates/codestory-store/src/** - - crates/codestory-cli/src/retrieval.rs - - crates/codestory-cli/src/main.rs - - crates/codestory-cli/src/args.rs - - crates/codestory-cli/src/runtime.rs - - crates/codestory-cli/src/stdio_*.rs - - crates/codestory-cli/tests/fixtures/packet_search_eval/** - - crates/codestory-cli/tests/packet_search_eval.rs - - crates/codestory-cli/tests/retrieval_bootstrap_contracts.rs - - crates/codestory-cli/tests/search_json_output.rs - - crates/codestory-cli/tests/stdio_protocol_contracts.rs - - crates/codestory-runtime/src/** - - crates/codestory-indexer/Cargo.toml - - crates/codestory-indexer/src/lib.rs - - scripts/lint-retrieval-generalization.mjs - - scripts/codestory-release-evidence-gate.mjs - - scripts/tests/codestory-release-evidence-gate.test.mjs - - benchmarks/release-evidence/** - - scripts/**retrieval** + - .codex/environments/** + - .cursor/rules/** + - .github/actions/** + - .github/scripts/** + - .github/workflows/** + - benchmarks/** + - crates/** + - docker/** + - docs/architecture/language-support.md + - docs/architecture/retrieval-* + - docs/contributors/testing-matrix.md - docs/ops/retrieval-sidecars.md - - docs/architecture/retrieval-*.md + - docs/testing/codestory-e2e-stats-log.md + - docs/testing/performance-review-playbook.md - docs/testing/retrieval-architecture.md - - docker/retrieval-compose.yml - - .github/workflows/retrieval-sidecar-smoke.yml + - plugins/codestory/** + - scripts/** # Base-branch runs seed caches that sibling PRs are allowed to restore. push: branches: - main - dev/codestory-next paths: - - crates/codestory-retrieval/** - - crates/codestory-contracts/** - - crates/codestory-store/Cargo.toml - - crates/codestory-store/src/** - - crates/codestory-cli/src/retrieval.rs - - crates/codestory-cli/src/main.rs - - crates/codestory-cli/src/args.rs - - crates/codestory-cli/src/runtime.rs - - crates/codestory-cli/src/stdio_*.rs - - crates/codestory-cli/tests/fixtures/packet_search_eval/** - - crates/codestory-cli/tests/packet_search_eval.rs - - crates/codestory-cli/tests/retrieval_bootstrap_contracts.rs - - crates/codestory-cli/tests/search_json_output.rs - - crates/codestory-cli/tests/stdio_protocol_contracts.rs - - crates/codestory-runtime/src/** - - crates/codestory-indexer/Cargo.toml - - crates/codestory-indexer/src/lib.rs - - scripts/lint-retrieval-generalization.mjs - - scripts/codestory-release-evidence-gate.mjs - - scripts/tests/codestory-release-evidence-gate.test.mjs - - benchmarks/release-evidence/** - - scripts/**retrieval** + - .codex/environments/** + - .cursor/rules/** + - .github/actions/** + - .github/scripts/** + - .github/workflows/** + - benchmarks/** + - crates/** + - docker/** + - docs/architecture/language-support.md + - docs/architecture/retrieval-* + - docs/contributors/testing-matrix.md - docs/ops/retrieval-sidecars.md - - docs/architecture/retrieval-*.md + - docs/testing/codestory-e2e-stats-log.md + - docs/testing/performance-review-playbook.md - docs/testing/retrieval-architecture.md - - docker/retrieval-compose.yml - - .github/workflows/retrieval-sidecar-smoke.yml + - plugins/codestory/** + - scripts/** workflow_dispatch: concurrency: @@ -115,6 +95,9 @@ jobs: restore-keys: | ${{ runner.os }}-cargo-stable-${{ steps.rust-cache-key.outputs.version }}-${{ steps.rust-cache-key.outputs.target }}-retrieval-contracts-default-features- + - name: Generalization lint regression contracts + run: cargo test -p codestory-runtime --test retrieval_generalization_guard + - name: Runtime sidecar and packet contract tests run: | cargo test -p codestory-runtime --lib agent::retrieval_primary::tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 94d02f76..d1319bb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,23 @@ project/workspace identity differs from the selected repository. `cache identity` now exposes the lossless project, workspace, artifact, and safe legacy-alias disposition used by that verification. +- Separated release-evidence provenance validation from the benchmark corpus + loader and expanded the generalization guard across the repository-controlled + non-Rust product and release-control boundary. Direct and adjacent/split + evaluation-corpus dependencies now fail in plugin launch/setup surfaces, + worktree/install tooling, runtime configuration, release scripts/workflows, + and the release evaluator across native separators, case variants, + direct harness commands/configuration, comment-separated JavaScript imports, + unquoted shell continuations, workflow-embedded shell continuations, and + PowerShell continuations. Comment-only JavaScript and hash-commented + shell/config text is ignored, while Markdown prose stays outside evaluation + and dependency scanning. The CI trigger and focused regression contract cover + the full protected inventory, including local composite actions and shipped + agent instruction surfaces, and missing protected paths fail closed. Exact + release-policy references to the retrieval smoke workflow are allowed only + by file, literal, and use; every direct or constructed occurrence is checked, + and constructed dependencies must span the literals that form the matched + harness marker. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 7ad95b11..7b3bd04b 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -131,6 +131,41 @@ fn run_lint_with_prompt_script_fixture(contents: &str) -> Output { .expect("run lint with prompt script fixture") } +fn run_lint_with_non_rust_fixtures(fixtures: &[(&str, &str)]) -> Output { + let repo_root = workspace_root(); + let script = lint_script(&repo_root); + let fixture_root = TempDir::new().expect("create fixture root"); + std::fs::write( + fixture_root.path().join("neutral.rs"), + "pub fn repository_neutral_fixture() {}\n", + ) + .expect("write neutral Rust fixture"); + for (file_name, contents) in fixtures { + let file_path = fixture_root.path().join(file_name); + std::fs::create_dir_all(file_path.parent().expect("fixture parent")) + .expect("create non-Rust fixture parent"); + std::fs::write(file_path, contents).expect("write non-Rust fixture"); + } + + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("lock lint script subprocess"); + Command::new("node") + .arg(&script) + .current_dir(&repo_root) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", + fixture_root.path(), + ) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_NON_RUST_SCAN_ROOTS", + fixture_root.path(), + ) + .output() + .expect("run lint with non-Rust fixture") +} + #[test] fn retrieval_generalization_lint_script_exits_clean_with_extra_fixture_root() { let repo_root = workspace_root(); @@ -386,6 +421,215 @@ pub const LEAKED_DEPENDENCY: &str = include_str!(concat!("../../benchmarks/", "t } } +#[test] +fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { + let rejected = run_lint_with_non_rust_fixtures(&[ + ( + "leaked.ps1", + "$corpus = \"scripts\\cross-repo-\" + `\n \"sourcetrail-queries.mjs\"\n", + ), + ( + "leaked.sh", + "prefix=./scripts\nscript=${prefix#./}/fetch-holdout-repos.mjs\ncorpus=benchmarks/ta\\\nsks/eval-probes.json\n", + ), + ( + "workflow-command.yml", + "run: |2-\n node scripts/fetch-\\\n holdout-repos.mjs\n", + ), + ( + "surrounding-command.mjs", + "const command = \"node scripts/fetch-\" + \"holdout-repos.mjs --json\";\nconst config = \"prefix benchmarks/ta\" + \"sks/eval-probes.json suffix\";\n", + ), + ( + "line-continuation.mjs", + "const script = \"scripts/fetch-holdout-\\\nrepos.mjs\";\n", + ), + ( + "joined-shell-word.sh", + "node scripts/fetch-'holdout-repos.mjs'\n", + ), + ( + "joined-workflow-word.yml", + "run: |\n node scripts/fetch-'holdout-repos.mjs'\n", + ), + ( + "quoted-run-key.yml", + "steps:\n - \"run\": |\n node scripts/fetch-'holdout-repos.mjs'\n", + ), + ( + "escaped-shell-word.sh", + "node scripts/fetch\\-holdout-repos.mjs\n", + ), + ( + "quoted-yaml-scalar.yml", + "value: 'clean # scripts/fetch-holdout-repos.mjs'\n", + ), + ( + "quoted-block-scalar.yml", + "run: |\n value='clean # scripts/fetch-holdout-repos.mjs'\n", + ), + ( + "github-script.yml", + "uses: actions/github-script@v8\nwith:\n script: |\n const script = \"scripts/fetch-\" + \"holdout-repos.mjs\";\n", + ), + ( + "direct-harness-import.mjs", + "import \"./scripts/codestory-agent-ab-benchmark.mjs\";\n", + ), + ( + "constructed-harness-import.mjs", + "const harness = \"scripts/codestory-agent-ab-\" + \"benchmark.mjs\";\nawait import(harness);\n", + ), + ( + "unapproved-policy-reference.mjs", + "const workflows = [\".github/workflows/retrieval-sidecar-smoke.yml\", \"unrelated.yml\"];\n", + ), + ( + "plugins/codestory/skills/codestory-grounding/SKILL.md", + "Run `node scripts/fetch-holdout-repos.mjs` before grounding.\n", + ), + ( + ".cursor/rules/codestory.mdc", + "Read benchmarks/tasks/eval-probes.json before answering.\n", + ), + ]); + let rejected_stderr = String::from_utf8_lossy(&rejected.stderr).to_ascii_lowercase(); + assert!( + !rejected.status.success(), + "executable corpus dependencies must fail lint; stderr={rejected_stderr}" + ); + for (file_name, marker) in [ + ("leaked.ps1", "scriptscrossreposourcetrailqueriesmjs"), + ("leaked.sh", "benchmarkstasksevalprobesjson"), + ("workflow-command.yml", "fetchholdoutreposmjs"), + ("surrounding-command.mjs", "fetchholdoutreposmjs"), + ("line-continuation.mjs", "fetchholdoutreposmjs"), + ("joined-shell-word.sh", "fetch-'holdout-repos.mjs"), + ("joined-workflow-word.yml", "fetch-'holdout-repos.mjs"), + ("quoted-run-key.yml", "fetch-'holdout-repos.mjs"), + ("escaped-shell-word.sh", "fetch\\-holdout-repos.mjs"), + ("quoted-yaml-scalar.yml", "fetch-holdout-repos.mjs"), + ("quoted-block-scalar.yml", "fetch-holdout-repos.mjs"), + ("github-script.yml", "fetchholdoutreposmjs"), + ( + "direct-harness-import.mjs", + "codestory-agent-ab-benchmark.mjs", + ), + ( + "constructed-harness-import.mjs", + "codestoryagentabbenchmarkmjs", + ), + ( + "unapproved-policy-reference.mjs", + "retrieval-sidecar-smoke.yml", + ), + ("skill.md", "fetch-holdout-repos.mjs"), + ("codestory.mdc", "benchmarks/tasks"), + ] { + assert!( + rejected_stderr.contains(file_name) && rejected_stderr.contains(marker), + "lint failure should identify {file_name} and {marker}; stderr={rejected_stderr}" + ); + } + + let allowed = run_lint_with_non_rust_fixtures(&[ + ( + "prose.md", + "The benchmark harness reads `benchmarks/tasks/eval-probes.json`; production code must not.\n", + ), + ( + "quoted-shell.sh", + "value='scripts/fetch-\\\nholdout-repos.mjs'\n", + ), + ( + "unrelated-list.yml", + "- scripts/fetch-\\\n- holdout-repos.mjs\n", + ), + ( + "template-comment.mjs", + "const value = `${({ clean: true }).clean /* scripts/fetch-holdout-repos.mjs */}`;\n", + ), + ( + "quoted-shell-comment.sh", + "value='clean\\' # scripts/fetch-holdout-repos.mjs\n", + ), + ( + "quoted-powershell-comment.ps1", + "$value = 'clean`' # scripts/fetch-holdout-repos.mjs\n", + ), + ( + "quoted-yaml-comment.yml", + "value: 'clean\\' # scripts/fetch-holdout-repos.mjs\n", + ), + ( + "folded-workflow.yml", + "run: >-\n node scripts/fetch-\\\n holdout-repos.mjs\n", + ), + ( + "comment-only.yml", + "# run: node scripts/fetch-\\\n# holdout-repos.mjs\nrun: echo clean\n", + ), + ( + "plain-apostrophe.yml", + "message: don't load it # scripts/fetch-holdout-repos.mjs\n", + ), + ( + "punctuated-apostrophe.yml", + "message: rock-'n roll # scripts/fetch-holdout-repos.mjs\n", + ), + ( + "doubled-single-quote.yml", + "value: 'scripts/fetch-''holdout-repos.mjs'\n", + ), + ]); + let allowed_stderr = String::from_utf8_lossy(&allowed.stderr); + assert!( + allowed.status.success(), + "prose, comments, and unrelated continuations must pass lint; stderr={allowed_stderr}" + ); +} + +#[test] +fn linter_binds_policy_allowances_to_the_exact_approved_use() { + let allowed = run_lint_with_non_rust_fixtures(&[ + ( + ".github/scripts/route-ci-proof.mjs", + " \".github/workflows/retrieval-sidecar-smoke.yml\",\n", + ), + ( + ".github/scripts/check-workflow-policy.mjs", + "const retrievalSidecarSmoke = path.join(workflowRoot, \"retrieval-sidecar-smoke.yml\");\nviolations.push(\"retrieval-sidecar-smoke.yml must exist\");\nviolations.push(\"retrieval-sidecar-smoke.yml Windows proof must be workflow_dispatch-only\");\n", + ), + ]); + let allowed_stderr = String::from_utf8_lossy(&allowed.stderr); + assert!( + allowed.status.success(), + "the exact policy and routing references must pass lint; stderr={allowed_stderr}" + ); + + let rejected = run_lint_with_non_rust_fixtures(&[ + ( + ".github/scripts/route-ci-proof.mjs", + " \".github/workflows/retrieval-sidecar-smoke.yml\",\n \".github/workflows/retrieval-sidecar-smoke.yml\",\nawait import(\".github/workflows/retrieval-sidecar-smoke.yml\");\n", + ), + ( + ".github/scripts/check-workflow-policy.mjs", + "const retrievalSidecarSmoke = path.join(workflowRoot, \"retrieval-sidecar-smoke.yml\");\nconst hostile = \".github/workflows/retrieval-\" + \"sidecar-smoke.yml\";\n", + ), + ]); + let rejected_stderr = String::from_utf8_lossy(&rejected.stderr).to_ascii_lowercase(); + assert!( + !rejected.status.success(), + "an approved file must not hide another harness use; stderr={rejected_stderr}" + ); + assert!( + rejected_stderr.contains("route-ci-proof.mjs:3:await import") + && rejected_stderr.contains("check-workflow-policy.mjs:2:") + && rejected_stderr.contains("githubworkflowsretrievalsidecarsmokeyml"), + "lint must identify both the hostile direct and split uses; stderr={rejected_stderr}" + ); +} + #[test] fn linter_fails_closed_when_one_prompt_corpus_entry_is_not_a_literal() { let output = run_lint_with_prompt_script_fixture( diff --git a/docs/testing/codestory-e2e-stats-log.md b/docs/testing/codestory-e2e-stats-log.md index 79bbe338..5c096955 100644 --- a/docs/testing/codestory-e2e-stats-log.md +++ b/docs/testing/codestory-e2e-stats-log.md @@ -114,6 +114,7 @@ Rows whose commit cell ends in `+wt` were run from the working tree based on tha | 2026-07-13 | 0c2d830d+wt | pass, final macOS support pre-PR full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded the latest stats-log baseline by more than 25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1 because CODESTORY_REAL_REPO_DRILL_CASES was not set; checksum-matched native llama-server with explicit CPU allowance for the stats lane; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; symbol_search_docs 19,225; dense anchors 1,006; dense skips 19,225; retrieval_index_seconds 22.90; retrieval_status_seconds 1.03; repeat full refresh 15.00s with 0 embedded | 15.55 | 0.11 | 2.18 | 0.28 | 0.11 | 0.10 | 148,236 | 123,652 | 307 | 0 | 1,006 | true | | 2026-07-13 | a2e34238+wt | pass, PR #1048 review remediation full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded the latest stats-log baseline by more than 25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1 because CODESTORY_REAL_REPO_DRILL_CASES was not set; checksum-matched native llama-server with explicit CPU allowance for the stats lane; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; symbol_search_docs 19,246; dense anchors 1,006; dense skips 19,246; retrieval_index_seconds 22.02; retrieval_status_seconds 0.96; repeat full refresh 15.12s with 0 embedded | 15.94 | 0.10 | 2.11 | 0.24 | 0.09 | 0.08 | 148,383 | 123,783 | 307 | 0 | 1,006 | true | | 2026-07-13 | 72c674e5+1050wt | pass, issue #1050 same-timestamp content-fingerprint freshness full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded the latest stats-log baseline by more than 25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1 because CODESTORY_REAL_REPO_DRILL_CASES was not set; checksum-matched native llama-server with explicit CPU allowance for the stats lane; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; symbol_search_docs 19,251; dense anchors 1,007; dense skips 19,251; retrieval_index_seconds 31.65; retrieval_status_seconds 1.04; repeat full refresh 17.86s with 0 embedded | 19.72 | 0.18 | 2.44 | 0.31 | 0.18 | 0.14 | 148,437 | 123,824 | 307 | 0 | 1,007 | true | +| 2026-07-13 | 9c97808a+1054wt | pass, issue #1054 non-Rust corpus-boundary guard full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded the latest stats-log baseline by more than 25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1 because CODESTORY_REAL_REPO_DRILL_CASES was not set; checksum-matched native llama-server with explicit CPU allowance for the stats lane; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; symbol_search_docs 19,257; dense anchors 1,007; dense skips 19,257; retrieval_index_seconds 66.14; retrieval_status_seconds 0.91; repeat full refresh 16.78s with 0 embedded | 17.58 | 0.14 | 2.23 | 0.27 | 0.12 | 0.11 | 148,577 | 123,886 | 308 | 0 | 1,007 | true | ## Repeat And Report Timing @@ -164,6 +165,7 @@ gates. | 2026-07-13 | 0c2d830d+wt | final macOS support pre-PR full-sidecar stats on Apple Silicon; proof_tier full_sidecar; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; repeat graph 7.75s; repeat semantic 0.42s; repeat cache/search projection/index 0.00s/0.20s/1.86s | 15.00 | 1.79 | 0.78 | 1.02 | | 2026-07-13 | a2e34238+wt | PR #1048 review remediation full-sidecar stats on Apple Silicon; proof_tier full_sidecar; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; repeat graph 7.89s; repeat semantic 0.48s; repeat cache/search projection/index 0.00s/0.19s/1.84s | 15.12 | 1.68 | 0.73 | 0.95 | | 2026-07-13 | 72c674e5+1050wt | issue #1050 same-timestamp content-fingerprint freshness full-sidecar stats on Apple Silicon; proof_tier full_sidecar; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; repeat graph 8.89s; repeat semantic 0.62s; repeat cache/search projection/index 0.00s/0.23s/1.84s | 17.86 | 2.23 | 0.97 | 1.26 | +| 2026-07-13 | 9c97808a+1054wt | issue #1054 non-Rust corpus-boundary guard full-sidecar stats on Apple Silicon; proof_tier full_sidecar; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; repeat graph 8.64s; repeat semantic 0.53s; repeat cache/search projection/index 0.00s/0.23s/1.82s | 16.78 | 1.90 | 0.82 | 1.08 | ## Phase Metrics @@ -271,6 +273,7 @@ from this phase table rather than backfilled. | 2026-07-13 | a2e34238+wt | PR #1048 review remediation full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 22.02; retrieval_status_seconds 0.96; repeat full refresh 15.12s with 0 embedded; report_seconds 1.68; symbol_search_docs 19,246; dense anchors 1,006; reasons public_api 934, entrypoint 12, central_graph_node 48, component_report 12 | 15.94 | 7.85 | 0.50 | 0 | 0 | 0 | | 2026-07-13 | 72c674e5+1050wt | issue #1050 same-timestamp content-fingerprint freshness full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 31.65; retrieval_status_seconds 1.04; repeat full refresh 17.86s with 0 embedded; report_seconds 2.23; symbol_search_docs 19,251; dense anchors 1,007; reasons public_api 935, entrypoint 12, central_graph_node 48, component_report 12 | 19.72 | 10.28 | 0.59 | 0 | 0 | 0 | | 2026-07-13 | 72c674e5+1035wt | issue #1035 request-scoped filesystem-aware plugin routing full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warnings index_seconds and semantic_phase_seconds exceeded the latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 60.11; retrieval_status_seconds 0.82; repeat full refresh 17.20s with 0 embedded; report_seconds 1.84; symbol_search_docs 19,258; dense anchors 1,006; post-rebase focused identity tests and workspace check passed on 9c97808a | 20.02 | 0.12 | 2.19 | 0.26 | 0.10 | 0.09 | 148,419 | 123,802 | 307 | 0 | 1,006 | true | +| 2026-07-13 | 9c97808a+1054wt | issue #1054 non-Rust corpus-boundary guard full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 66.14; retrieval_status_seconds 0.91; repeat full refresh 16.78s with 0 embedded; report_seconds 1.90; symbol_search_docs 19,257; dense anchors 1,007; reasons public_api 935, entrypoint 12, central_graph_node 48, component_report 12 | 17.58 | 8.35 | 0.65 | 0 | 0 | 0 | | 2026-07-13 | 9c97808a+1053wt | issue #1053 grounding-only managed plugin convergence full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 56.30; retrieval_status_seconds 0.87; repeat full refresh 15.44s with 0 embedded; report_seconds 1.81; symbol_search_docs 19,253; dense anchors 1,007; reasons public_api 935, entrypoint 12, central_graph_node 48, component_report 12 | 15.21 | 0.13 | 2.22 | 0.28 | 0.12 | 0.12 | 148,476 | 123,857 | 307 | 0 | 1,007 | true | | 2026-07-13 | a4d3d6d4+1058wt | PR #1058 managed convergence review remediation full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 48.67; retrieval_status_seconds 0.83; repeat full refresh 15.36s with 0 embedded; report_seconds 1.84; symbol_search_docs 19,268; dense anchors 1,007; reasons public_api 935, entrypoint 12, central_graph_node 48, component_report 12 | 15.83 | 0.14 | 2.16 | 0.28 | 0.12 | 0.12 | 148,550 | 123,901 | 307 | 0 | 1,007 | true | | 2026-07-13 | 4e8a94f7 | PR #1058 promoted-head full-sidecar stats on hosted Ubuntu; proof_tier full_sidecar; warnings index_seconds, semantic_phase_seconds, retrieval_index_seconds, and search_seconds exceeded the Apple Silicon living baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 195.05; retrieval_status_seconds 1.14; repeat full refresh 35.45s with 0 embedded; report_seconds 3.70; symbol_search_docs 19,279; dense anchors 1,007; reasons public_api 935, entrypoint 12, central_graph_node 48, component_report 12 | 37.09 | 0.12 | 4.19 | 0.52 | 0.09 | 0.09 | 148,627 | 123,968 | 307 | 0 | 1,007 | true | diff --git a/docs/testing/performance-review-playbook.md b/docs/testing/performance-review-playbook.md index c43fea58..2d4b1a64 100644 --- a/docs/testing/performance-review-playbook.md +++ b/docs/testing/performance-review-playbook.md @@ -165,12 +165,40 @@ ordinary CI but are explicitly `release_eligible: false`. A product profile must be created from provisioned raw evidence and explicitly approved before a release workflow can adopt this gate. -The corpus boundary is deliberately precise: it scans Rust production files in -all non-benchmark crates for direct path strings and adjacent/split literal -construction such as `concat!` and `include_str!`. It does not prove arbitrary -runtime path generation, non-Rust production surfaces, environment/config -indirection, or dependencies hidden behind external processes. Those surfaces -remain explicit follow-up audit scope rather than being claimed as covered. +The corpus boundary is deliberately precise. The generalization lint scans Rust +production files in all non-benchmark crates for copied corpus content, direct +paths, and adjacent/split literal construction. It also scans the following +repository-controlled non-Rust surfaces for direct and adjacent/split +dependencies on every inventoried evaluation/query corpus: + +The inventory is executable rather than documentation-only. Supported text and +configuration files under `scripts/`, `.github/scripts/`, +`.github/workflows/`, the shipped plugin, retrieval Compose configuration, and +native backend metadata enter the protected scan by default. The tracked Codex +environment definition is a required protected file. Only the explicit +corpus/proof harness list, the named release-detector unit-test harness, and +test/fixture directories are excluded; there are no blanket test-filename +exceptions. Missing required protected or harness paths fail the lint so moves +require review. + +| Classification | Protected or allowed surface | Static contract | +| --- | --- | --- | +| Product launch and setup | Plugin manifests, hooks, MCP launcher, grounding skill guidance/setup scripts, Codex environment/worktree setup, and the Windows installer | Protected; the configured directories are scanned recursively, while test and fixture directories are excluded. | +| Runtime configuration | Retrieval Compose configuration and native sidecar backend metadata | Protected; new supported text/config files under those directories enter the scan automatically. | +| Release control | Release/auto-release workflows, version detection/checking, package assembly, the release-evidence evaluator, and shared provenance validation | Protected; required files must exist, and protected non-Rust modules cannot import an explicitly classified corpus/proof harness module. | +| Explicit corpus and test harnesses | Task manifests, packet/repo benchmark drivers and scorers, holdout provisioning, release-candidate measurement, the retrieval-sidecar contract workflow, the release-detector unit test, and test fixtures | Allowed to load evaluation corpora or exercise protected code; these paths are evidence producers/tests, not product or release-decision logic. Other scripts and workflows remain protected by default. | +| Environment and generated inputs | Environment values, downloaded manifests/models, generated evidence, and workflow artifacts | Static code paths are protected where listed above. Runtime values and generated bytes require the existing schema, hash, identity, and provenance checks. | +| External processes | Git, Docker/Qdrant, embedding servers, and agent executables | Static command construction is protected where it is part of a listed surface. The external executable's internal behavior is outside this repository scan. | + +The release evaluator now imports repository/cache provenance checks from +`scripts/codestory-evidence-provenance.mjs`, a corpus-neutral module shared with +the benchmark harness. It no longer loads the packet benchmark module—and its +query catalog—as a transitive release-control dependency. + +This guard does not prove arbitrary runtime string generation, environment +values, generated content, or external-process behavior. Those boundaries stay +fail closed through the release-evidence attestation/runtime checks where a +machine-verifiable contract exists; otherwise they remain explicit non-claims. Do not promote importable or rebuildable graph/sidecar artifacts in this slice. A follow-up PR for that idea must require provenance before reuse: source root, diff --git a/docs/testing/retrieval-architecture.md b/docs/testing/retrieval-architecture.md index 44e4d412..10b4ca45 100644 --- a/docs/testing/retrieval-architecture.md +++ b/docs/testing/retrieval-architecture.md @@ -158,7 +158,7 @@ Non-claims: | CLI lifecycle | `codestory-cli` `retrieval up\|down\|status\|index\|query` | Local data dirs, health JSON, standalone query | | Packet integration | `codestory-runtime/src/agent/retrieval_primary.rs` | Primary sidecar path, diagnostic traces, promotion warnings | | Nucleo policy | `codestory-runtime/src/agent/nucleo_policy.rs` | Suppresses Nucleo O(n) scan on sidecar primary; disabled sidecars are not valid product evidence | -| Generalization lint | `scripts/lint-retrieval-generalization.mjs` | Derives banned identities, prompts, claims, paths, and query/probe phrases from benchmark manifests, script prompt/query catalogs, and the eval-only probe manifest/source, then scans Rust production retrieval trees after masking test-only items (CI via Rust guard test); missing, malformed, or partially parsed corpora fail closed | +| Generalization lint | `scripts/lint-retrieval-generalization.mjs` | Derives banned identities, prompts, claims, paths, and query/probe phrases from benchmark manifests, script prompt/query catalogs, and the eval-only probe manifest/source. It scans Rust production retrieval trees after masking test-only items and structurally scans the inventoried non-Rust product/release-control boundary for direct or adjacent/split corpus dependencies (CI via the Rust guard test); missing protected paths and missing, malformed, or partially parsed corpora fail closed. | All planned retrieval stages use the same fixed-capacity worker pool, including symbol-like and natural-language queries. Each job carries the request deadline diff --git a/scripts/codestory-agent-ab-benchmark.mjs b/scripts/codestory-agent-ab-benchmark.mjs index df100778..bebfa279 100644 --- a/scripts/codestory-agent-ab-benchmark.mjs +++ b/scripts/codestory-agent-ab-benchmark.mjs @@ -22,6 +22,12 @@ import { shouldPrepareRetrievalIndex, unsupportedSidecarContractRequests, } from "./codestory-benchmark-contract.mjs"; +import { + cacheProvenanceBlockers, + isImmutableCommitRef, + isTrustedPublishableRepoUrl, + repoProvenanceBlockers, +} from "./codestory-evidence-provenance.mjs"; const scriptDir = path.dirname(fileURLToPath(import.meta.url)); const benchmarkHarnessPath = fileURLToPath(import.meta.url); @@ -1012,39 +1018,6 @@ function uniqueTaskRepos(tasks) { return repos; } -function isTrustedPublishableRepoUrl(url) { - try { - const parsed = new URL(String(url ?? "")); - if ( - parsed.protocol !== "https:" || - parsed.hostname.toLowerCase() !== "github.com" || - parsed.username || - parsed.password || - parsed.search || - parsed.hash - ) { - return false; - } - const parts = parsed.pathname.split("/").filter(Boolean); - return ( - parts.length === 2 && - /^[A-Za-z0-9_.-]+$/.test(parts[0]) && - /^[A-Za-z0-9_.-]+(?:\.git)?$/.test(parts[1]) - ); - } catch { - return false; - } -} - -function normalizeTrustedPublishableRepoUrl(url) { - if (!isTrustedPublishableRepoUrl(url)) { - return null; - } - const parsed = new URL(String(url)); - const [owner, repo] = parsed.pathname.split("/").filter(Boolean); - return `${owner.toLowerCase()}/${repo.replace(/\.git$/i, "").toLowerCase()}`; -} - function manifestRepoMaterializationBlockers(tasks, opts = {}) { if (!opts.publishable || !opts.materializeRepos) { return []; @@ -4746,118 +4719,6 @@ function packetRuntimeMarkdownRow(row) { return `| ${cells.join(" | ")} |`; } -function repoProvenanceBlockers(result) { - const provenance = result.repo_provenance; - if (!provenance) { - return ["missing repo provenance"]; - } - const reasons = []; - if (provenance.manifest_overridden_by_builtin) { - reasons.push("manifest repo was overridden by a built-in checkout"); - } - const configuredRef = provenance.configured?.ref ?? null; - const manifestRef = provenance.manifest?.ref ?? null; - const configuredCommit = normalizeImmutableCommitRef(configuredRef); - const manifestCommit = manifestRef ? normalizeImmutableCommitRef(manifestRef) : null; - const gitHead = normalizeImmutableCommitRef(provenance.git_head); - if (!configuredCommit) { - reasons.push("repo ref is not pinned to a full immutable commit SHA"); - } - if (manifestRef && configuredRef && manifestCommit !== configuredCommit) { - reasons.push(`manifest ref ${manifestRef} does not match configured ref ${configuredRef}`); - } - if (!gitHead) { - reasons.push("missing git head"); - } else if (configuredCommit && gitHead !== configuredCommit) { - reasons.push(`git head ${provenance.git_head} does not match configured ref ${configuredRef}`); - } - const configuredUrl = provenance.configured?.url ?? null; - const manifestUrl = provenance.manifest?.url ?? null; - const gitOrigin = provenance.git_origin ?? null; - const configuredRepo = normalizeTrustedPublishableRepoUrl(configuredUrl); - const manifestRepo = manifestUrl ? normalizeTrustedPublishableRepoUrl(manifestUrl) : null; - const originRepo = gitOrigin ? normalizeTrustedPublishableRepoUrl(gitOrigin) : null; - if (!configuredRepo) { - reasons.push("configured repo URL is not a trusted GitHub HTTPS repo URL"); - } - if (!manifestUrl) { - reasons.push("missing manifest repo URL"); - } else if (!manifestRepo) { - reasons.push("manifest repo URL is not a trusted GitHub HTTPS repo URL"); - } - if (configuredRepo && manifestUrl && manifestRepo && manifestRepo !== configuredRepo) { - reasons.push(`manifest repo URL ${manifestUrl} does not match configured URL ${configuredUrl}`); - } - if (!originRepo) { - reasons.push("git origin is missing or is not a trusted GitHub HTTPS repo URL"); - } else if (configuredRepo && originRepo !== configuredRepo) { - reasons.push(`git origin ${gitOrigin} does not match configured URL ${configuredUrl}`); - } - if (provenance.git_dirty !== false) { - reasons.push(provenance.git_dirty ? "repo checkout is dirty" : "repo cleanliness is unknown"); - } - return reasons; -} - -function isImmutableCommitRef(ref) { - return /^[0-9a-f]{40}$/i.test(String(ref ?? "").trim()); -} - -function normalizeImmutableCommitRef(ref) { - const value = String(ref ?? "").trim(); - return isImmutableCommitRef(value) ? value.toLowerCase() : null; -} - -function cacheProvenanceBlockers(result) { - const provenance = result.codestory_cache_provenance; - if (!provenance) { - return ["missing CodeStory cache provenance"]; - } - const reasons = []; - if (provenance.doctor_status !== "pass") { - reasons.push("CodeStory doctor provenance failed"); - } - if (!provenance.storage_path) { - reasons.push("missing CodeStory cache path"); - } - if (!provenance.cache_policy) { - reasons.push("missing CodeStory cache policy"); - } - if (provenance.cache_policy === "unprepared-cache-blocked") { - reasons.push("CodeStory sidecar cache was not prepared"); - } - if (provenance.retrieval_mode !== "full") { - reasons.push(`CodeStory retrieval mode=${provenance.retrieval_mode ?? "unknown"}; expected full`); - } - if (!provenance.sidecar_generation) { - reasons.push("missing CodeStory sidecar generation"); - } - if (provenance.manifest_embedding_backend !== "llamacpp:bge-base-en-v1.5") { - reasons.push( - `CodeStory sidecar embedding backend=${provenance.manifest_embedding_backend ?? "unknown"}; expected llamacpp:bge-base-en-v1.5`, - ); - } - if (provenance.semantic_backend == null) { - reasons.push("missing CodeStory semantic backend"); - } - if (provenance.local_only !== true) { - reasons.push(`CodeStory local-only guarantee is not proven (${provenance.locality_kind ?? "unknown"})`); - } - if (provenance.indexed !== true) { - reasons.push("CodeStory cache is not indexed"); - } - if (provenance.freshness_status !== "fresh") { - reasons.push(`CodeStory cache freshness=${provenance.freshness_status ?? "unknown"}`); - } - if (provenance.semantic_ready !== true) { - reasons.push("CodeStory semantic docs are not ready"); - } - if (provenance.indexing_in_timed_run == null) { - reasons.push("missing timed-run indexing provenance"); - } - return reasons; -} - function qualityFailureReasons(quality) { if (!quality) { return ["missing_quality_score"]; diff --git a/scripts/codestory-evidence-provenance.mjs b/scripts/codestory-evidence-provenance.mjs new file mode 100644 index 00000000..c640c3ba --- /dev/null +++ b/scripts/codestory-evidence-provenance.mjs @@ -0,0 +1,144 @@ +export function isTrustedPublishableRepoUrl(url) { + try { + const parsed = new URL(String(url ?? "")); + if ( + parsed.protocol !== "https:" + || parsed.hostname.toLowerCase() !== "github.com" + || parsed.username + || parsed.password + || parsed.search + || parsed.hash + ) { + return false; + } + const parts = parsed.pathname.split("/").filter(Boolean); + return ( + parts.length === 2 + && /^[A-Za-z0-9_.-]+$/.test(parts[0]) + && /^[A-Za-z0-9_.-]+(?:\.git)?$/.test(parts[1]) + ); + } catch { + return false; + } +} + +function normalizeTrustedPublishableRepoUrl(url) { + if (!isTrustedPublishableRepoUrl(url)) { + return null; + } + const parsed = new URL(String(url)); + const [owner, repo] = parsed.pathname.split("/").filter(Boolean); + return `${owner.toLowerCase()}/${repo.replace(/\.git$/i, "").toLowerCase()}`; +} + +export function isImmutableCommitRef(ref) { + return /^[0-9a-f]{40}$/i.test(String(ref ?? "").trim()); +} + +function normalizeImmutableCommitRef(ref) { + const value = String(ref ?? "").trim(); + return isImmutableCommitRef(value) ? value.toLowerCase() : null; +} + +export function repoProvenanceBlockers(result) { + const provenance = result.repo_provenance; + if (!provenance) { + return ["missing repo provenance"]; + } + const reasons = []; + if (provenance.manifest_overridden_by_builtin) { + reasons.push("manifest repo was overridden by a built-in checkout"); + } + const configuredRef = provenance.configured?.ref ?? null; + const manifestRef = provenance.manifest?.ref ?? null; + const configuredCommit = normalizeImmutableCommitRef(configuredRef); + const manifestCommit = manifestRef ? normalizeImmutableCommitRef(manifestRef) : null; + const gitHead = normalizeImmutableCommitRef(provenance.git_head); + if (!configuredCommit) { + reasons.push("repo ref is not pinned to a full immutable commit SHA"); + } + if (manifestRef && configuredRef && manifestCommit !== configuredCommit) { + reasons.push(`manifest ref ${manifestRef} does not match configured ref ${configuredRef}`); + } + if (!gitHead) { + reasons.push("missing git head"); + } else if (configuredCommit && gitHead !== configuredCommit) { + reasons.push(`git head ${provenance.git_head} does not match configured ref ${configuredRef}`); + } + const configuredUrl = provenance.configured?.url ?? null; + const manifestUrl = provenance.manifest?.url ?? null; + const gitOrigin = provenance.git_origin ?? null; + const configuredRepo = normalizeTrustedPublishableRepoUrl(configuredUrl); + const manifestRepo = manifestUrl ? normalizeTrustedPublishableRepoUrl(manifestUrl) : null; + const originRepo = gitOrigin ? normalizeTrustedPublishableRepoUrl(gitOrigin) : null; + if (!configuredRepo) { + reasons.push("configured repo URL is not a trusted GitHub HTTPS repo URL"); + } + if (!manifestUrl) { + reasons.push("missing manifest repo URL"); + } else if (!manifestRepo) { + reasons.push("manifest repo URL is not a trusted GitHub HTTPS repo URL"); + } + if (configuredRepo && manifestUrl && manifestRepo && manifestRepo !== configuredRepo) { + reasons.push(`manifest repo URL ${manifestUrl} does not match configured URL ${configuredUrl}`); + } + if (!originRepo) { + reasons.push("git origin is missing or is not a trusted GitHub HTTPS repo URL"); + } else if (configuredRepo && originRepo !== configuredRepo) { + reasons.push(`git origin ${gitOrigin} does not match configured URL ${configuredUrl}`); + } + if (provenance.git_dirty !== false) { + reasons.push(provenance.git_dirty ? "repo checkout is dirty" : "repo cleanliness is unknown"); + } + return reasons; +} + +export function cacheProvenanceBlockers(result) { + const provenance = result.codestory_cache_provenance; + if (!provenance) { + return ["missing CodeStory cache provenance"]; + } + const reasons = []; + if (provenance.doctor_status !== "pass") { + reasons.push("CodeStory doctor provenance failed"); + } + if (!provenance.storage_path) { + reasons.push("missing CodeStory cache path"); + } + if (!provenance.cache_policy) { + reasons.push("missing CodeStory cache policy"); + } + if (provenance.cache_policy === "unprepared-cache-blocked") { + reasons.push("CodeStory sidecar cache was not prepared"); + } + if (provenance.retrieval_mode !== "full") { + reasons.push(`CodeStory retrieval mode=${provenance.retrieval_mode ?? "unknown"}; expected full`); + } + if (!provenance.sidecar_generation) { + reasons.push("missing CodeStory sidecar generation"); + } + if (provenance.manifest_embedding_backend !== "llamacpp:bge-base-en-v1.5") { + reasons.push( + `CodeStory sidecar embedding backend=${provenance.manifest_embedding_backend ?? "unknown"}; expected llamacpp:bge-base-en-v1.5`, + ); + } + if (provenance.semantic_backend == null) { + reasons.push("missing CodeStory semantic backend"); + } + if (provenance.local_only !== true) { + reasons.push(`CodeStory local-only guarantee is not proven (${provenance.locality_kind ?? "unknown"})`); + } + if (provenance.indexed !== true) { + reasons.push("CodeStory cache is not indexed"); + } + if (provenance.freshness_status !== "fresh") { + reasons.push(`CodeStory cache freshness=${provenance.freshness_status ?? "unknown"}`); + } + if (provenance.semantic_ready !== true) { + reasons.push("CodeStory semantic docs are not ready"); + } + if (provenance.indexing_in_timed_run == null) { + reasons.push("missing timed-run indexing provenance"); + } + return reasons; +} diff --git a/scripts/codestory-release-evidence-gate.mjs b/scripts/codestory-release-evidence-gate.mjs index d3ceec3a..d9c89c52 100644 --- a/scripts/codestory-release-evidence-gate.mjs +++ b/scripts/codestory-release-evidence-gate.mjs @@ -9,7 +9,7 @@ import { pathToFileURL } from "node:url"; import { cacheProvenanceBlockers, repoProvenanceBlockers, -} from "./codestory-agent-ab-benchmark.mjs"; +} from "./codestory-evidence-provenance.mjs"; const METRICS = [ "status_seconds", "local_grounding_seconds", "convergence_seconds", diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 2ea3a2cc..9cf5a2f1 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -1,12 +1,11 @@ #!/usr/bin/env node /** - * CI guard: ban repo-specific path literals in retrieval integration production code. - * Scope is Rust production retrieval integration files. Benchmark/eval harness - * scripts and the env-gated eval probe module intentionally live outside this - * guard because their manifests name holdout repos; keep that boundary explicit - * instead of treating them as product code. - * Scans Rust files after masking `#[cfg(test)]` items/modules so test fixtures - * do not define the production contract. + * CI guard: keep production and release-control code independent from checked + * evaluation/query corpora. Rust production is checked for derived corpus + * content and structural paths after masking `#[cfg(test)]` items. Inventoried + * non-Rust product/release surfaces are checked for direct and adjacent/split + * corpus dependencies. Explicit benchmark/proof harnesses remain outside the + * protected scan because they must load those corpora. */ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; @@ -24,6 +23,147 @@ const explicitScanRoots = ( ) .split(path.delimiter) .filter(Boolean); +const explicitNonRustScanRoots = ( + process.env.CODESTORY_RETRIEVAL_GENERALIZATION_NON_RUST_SCAN_ROOTS ?? "" +) + .split(path.delimiter) + .filter(Boolean); + +const groundingSkillDir = path.join( + repoRoot, + "plugins", + "codestory", + "skills", + "codestory-grounding", +); + +const protectedNonRustDirs = [ + path.join(repoRoot, "scripts"), + path.join(repoRoot, ".github"), + path.join(repoRoot, ".cursor", "rules"), + path.join(repoRoot, "plugins", "codestory"), + path.join(repoRoot, "docker"), + path.join(repoRoot, "crates", "codestory-retrieval", "assets"), +]; + +const requiredProtectedNonRustFiles = [ + path.join(repoRoot, ".codex", "environments", "environment.toml"), + path.join(repoRoot, ".cursor", "rules", "codestory.mdc"), + path.join(groundingSkillDir, "SKILL.md"), + path.join(repoRoot, "scripts", "codestory-evidence-provenance.mjs"), + path.join(repoRoot, "scripts", "codestory-release-evidence-gate.mjs"), + path.join(repoRoot, "scripts", "codex-worktree-setup.mjs"), + path.join(repoRoot, "scripts", "codex-worktree-setup.ps1"), + path.join(repoRoot, "scripts", "codex-worktree-setup.sh"), + path.join(repoRoot, "scripts", "install-codestory.ps1"), + path.join(repoRoot, ".github", "scripts", "check-codestory-release.py"), + path.join(repoRoot, ".github", "scripts", "detect-codestory-release.py"), + path.join(repoRoot, ".github", "scripts", "package-codestory-release.py"), + path.join(repoRoot, ".github", "workflows", "auto-release.yml"), + path.join(repoRoot, ".github", "workflows", "release.yml"), +]; + +const corpusHarnessNonRustFiles = new Set([ + path.join(repoRoot, "scripts", "autoresearch-pipeline-score.mjs"), + path.join(repoRoot, "scripts", "codestory-agent-ab-benchmark.mjs"), + path.join(repoRoot, "scripts", "codestory-agent-ab-score.mjs"), + path.join(repoRoot, "scripts", "codestory-agent-value-score.mjs"), + path.join(repoRoot, "scripts", "codestory-benchmark-contract.mjs"), + path.join(repoRoot, "scripts", "codestory-language-holdout-integrity.mjs"), + path.join(repoRoot, "scripts", "codestory-manual-friction-check.mjs"), + path.join(repoRoot, "scripts", "cross-repo-sourcetrail-queries.mjs"), + path.join(repoRoot, "scripts", "fetch-holdout-repos.mjs"), + path.join(repoRoot, "scripts", "lint-retrieval-generalization.mjs"), + path.join(repoRoot, "scripts", "measure-peak-memory.ps1"), + path.join(repoRoot, "scripts", "prove-drill-packet-parity.mjs"), + path.join(repoRoot, "scripts", "score-drill-ledger.mjs"), + path.join(repoRoot, "scripts", "setup-retrieval-env.mjs"), + path.join(repoRoot, "scripts", "setup-retrieval-env.ps1"), + path.join(repoRoot, ".github", "scripts", "test-detect-codestory-release.py"), + path.join(repoRoot, ".github", "workflows", "release-candidate-evidence.yml"), + path.join(repoRoot, ".github", "workflows", "retrieval-sidecar-smoke.yml"), +].map((filePath) => path.resolve(filePath))); +const corpusSupportNonRustFiles = new Set([ + path.join(repoRoot, "scripts", "lint-retrieval-generalization.mjs"), + path.join(repoRoot, "scripts", "setup-retrieval-env.mjs"), + path.join(repoRoot, "scripts", "setup-retrieval-env.ps1"), + path.join(repoRoot, ".github", "scripts", "test-detect-codestory-release.py"), +].map((filePath) => path.resolve(filePath))); +const allowedHarnessReferences = [ + [ + path.join("plugins", "codestory", "skills", "codestory-grounding", "references", "drill-suite.md"), + "scripts/score-drill-ledger.mjs", + "`node scripts/score-drill-ledger.mjs [scored-report.json]`.", + ], + [ + path.join("plugins", "codestory", "skills", "codestory-grounding", "references", "retrieval-rollout.md"), + ".github/workflows/retrieval-sidecar-smoke.yml", + "| Smoke CI | `.github/workflows/retrieval-sidecar-smoke.yml` plus `docs/ops/retrieval-sidecars.md#preflight-smoke-contract` pass criteria | PRs touching retrieval crate, runtime/stdio/search wiring, indexer retrieval hooks, retrieval docs, scripts, Docker sidecar config, or the workflow | Full sidecar readiness. CI smoke uses `--skip-compose --wait-secs 0` and proves manifest-missing fail-closed shape only |", + ], + [ + path.join(".github", "scripts", "check-workflow-policy.mjs"), + "retrieval-sidecar-smoke.yml", + "const retrievalSidecarSmoke = path.join(workflowRoot, \"retrieval-sidecar-smoke.yml\");", + ], + [ + path.join(".github", "scripts", "check-workflow-policy.mjs"), + "retrieval-sidecar-smoke.yml", + "violations.push(\"retrieval-sidecar-smoke.yml must exist\");", + ], + [ + path.join(".github", "scripts", "check-workflow-policy.mjs"), + "retrieval-sidecar-smoke.yml", + "violations.push(\"retrieval-sidecar-smoke.yml Windows proof must be workflow_dispatch-only\");", + ], + [ + path.join(".github", "scripts", "route-ci-proof.mjs"), + ".github/workflows/retrieval-sidecar-smoke.yml", + "\".github/workflows/retrieval-sidecar-smoke.yml\",", + ], +].map(([relativePath, includes, use]) => ({ + relativePath, + includes, + use, +})); + +const executableJavaScriptExtensions = new Set([ + ".cjs", ".js", ".mjs", ".ts", ".tsx", +]); +const hashCommentExtensions = new Set([ + ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", +]); +const javaScriptStringOrCommentPattern = /("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*'|`(?:\\[\s\S]|[^`\\])*`)|(\/\*[\s\S]*?\*\/|\/\/[^\r\n]*)/g; +const javaScriptTemplateTokenPattern = /("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*')|(\$\{|[{}])|(\/\*[\s\S]*?\*\/|\/\/[^\r\n]*)/g; +const protectedNonRustExtensions = new Set([ + ...executableJavaScriptExtensions, + ".json", ".md", ".mdc", ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", +]); +const agentInstructionExtensions = new Set([".md", ".mdc"]); +const agentInstructionPathFragments = [ + path.join("plugins", "codestory", "skills", "codestory-grounding"), + path.join(".cursor", "rules"), +]; + +const defaultNonRustScanRoots = protectedNonRustDirs; +const usesDefaultNonRustScanRoots = explicitNonRustScanRoots.length === 0; +const nonRustScanRoots = usesDefaultNonRustScanRoots + ? defaultNonRustScanRoots + : explicitNonRustScanRoots.filter((root) => root && existsSync(root)); + +if (usesDefaultNonRustScanRoots) { + const missingProtectedPaths = [ + ...defaultNonRustScanRoots, + ...requiredProtectedNonRustFiles, + ...corpusHarnessNonRustFiles, + ].filter((requiredPath) => !existsSync(requiredPath)); + if (missingProtectedPaths.length > 0) { + console.error("lint-retrieval-generalization: missing protected non-Rust path(s)"); + for (const missingPath of missingProtectedPaths) { + console.error(` ${path.relative(repoRoot, missingPath)}`); + } + process.exit(2); + } +} const structuralScanDirs = readdirSync(path.join(repoRoot, "crates"), { withFileTypes: true }) .filter((entry) => entry.isDirectory() && entry.name !== "codestory-bench") @@ -117,6 +257,18 @@ if (scanDirs.length === 0 && productionOnlyFiles.length === 0) { process.exit(2); } +const evalCorpusBoundaryPatternList = evalCorpusBoundaryPatterns(); +const evalCorpusCompactPatternList = compactBoundaryPatterns( + evalCorpusBoundaryPatternList, +); +const corpusHarnessDependencyPatternList = corpusHarnessDependencyPatterns(); +const corpusHarnessDependencyRegexes = corpusHarnessDependencyPatternList.map( + (pattern) => new RegExp(`(?:^|[^a-z0-9_.-])${pattern}(?=$|[^a-z0-9_.-])`, "i"), +); +const corpusHarnessCompactPatternList = compactBoundaryPatterns( + corpusHarnessDependencyPatternList, +); + const bannedPatterns = [ "payload_config", "freelancer", @@ -206,7 +358,8 @@ const bannedPatterns = [ "nvm", "install\\.sh\\s+nvm", "bash_completion\\s+__nvm", - ...evalCorpusBoundaryPatterns(), + "--with-holdout-clone", + ...evalCorpusBoundaryPatternList, ...benchmarkManifestDerivedPatterns(), ...benchmarkEvalProbeDerivedPatterns(), ...benchmarkScriptPromptDerivedPatterns(), @@ -234,7 +387,7 @@ const bannedCompactPatterns = [ "datarequest", "sessiondelegate", "sourceanimatecss", - ...evalCorpusCompactPatterns(), + ...evalCorpusCompactPatternList, ]; const allowedPatternLines = [ @@ -262,11 +415,24 @@ function evalCorpusBoundaryPatterns() { return [ ...evalCorpusRoots.map((root) => path.relative(repoRoot, root).replaceAll(path.sep, "/")), ...corpusFiles.map((filePath) => path.relative(repoRoot, filePath).replaceAll(path.sep, "/")), + ...benchmarkIdentityScriptFiles.map((filePath) => path.basename(filePath)), ].map(escapeRegExp); } -function evalCorpusCompactPatterns() { - return evalCorpusBoundaryPatterns() +function corpusHarnessDependencyPatterns() { + const paths = new Set( + [...corpusHarnessNonRustFiles] + .filter((filePath) => !corpusSupportNonRustFiles.has(filePath)) + .flatMap((filePath) => [ + path.relative(repoRoot, filePath).replaceAll(path.sep, "/"), + path.basename(filePath), + ]), + ); + return [...paths].map(escapeRegExp); +} + +function compactBoundaryPatterns(boundaryPatterns) { + return boundaryPatterns .map((pattern) => compactProductionSource(pattern.replaceAll("\\", ""))) .filter((pattern) => pattern.length >= 12); } @@ -439,13 +605,13 @@ function addEvalProbeSourceQueryMarkers(markers, source) { let arrayMatch; while ((arrayMatch = queryArrayCall.exec(source)) != null) { const body = arrayMatch[1]; - const literals = rustStringLiteralSpans(body); + const literals = staticStringLiteralSpans(body); if (literals.length === 0 || rustArrayNonLiteralRemainder(body, literals).trim() !== "") { throw new Error("eval probe source query array contains an unparsed entry"); } for (const { literal } of literals) { parsedLiteralCount += 1; - addSpecificMarker(markers, rustStringLiteralContent(literal), { + addSpecificMarker(markers, staticStringLiteralContent(literal), { allowSpecificComposite: true, }); } @@ -558,6 +724,32 @@ function walkFiles(root, predicate) { return files; } +function walkProtectedNonRustFiles(root) { + return walkFiles(root, (filePath) => { + const extension = path.extname(filePath).toLowerCase(); + if (!protectedNonRustExtensions.has(extension)) { + return false; + } + if ( + agentInstructionExtensions.has(extension) + && !agentInstructionPathFragments.some((fragment) => + filePath.includes(`${path.sep}${fragment}${path.sep}`) + ) + ) return false; + if (!usesDefaultNonRustScanRoots) { + return true; + } + if (corpusHarnessNonRustFiles.has(path.resolve(filePath))) { + return false; + } + const segments = path.relative(repoRoot, filePath).split(path.sep); + return ( + !segments.includes("tests") + && !segments.includes("fixtures") + ); + }); +} + function addSpecificMarker(markers, value, options = {}) { if (typeof value !== "string") { return; @@ -1046,19 +1238,388 @@ function prepareProductionFile(filePath) { }; } +function prepareNonRustFile(filePath) { + const production = readFileSync(filePath, "utf8"); + const extension = path.extname(filePath).toLowerCase(); + const staticSource = maskNonRustComments(production, extension); + const lines = staticSource.split(/\r?\n/); + return { + filePath, + extension, + production, + lines, + logicalLines: logicalNonRustLines(lines, extension), + sourceLines: production.split(/\r?\n/), + staticSource, + literals: null, + }; +} + +function maskNonRustComments(source, extension) { + if (executableJavaScriptExtensions.has(extension)) { + return maskJavaScriptComments(source); + } + if (extension === ".yaml" || extension === ".yml") { + return maskYamlComments(source); + } + if (hashCommentExtensions.has(extension)) { + return maskHashComments(source, extension); + } + return source; +} + +function maskJavaScriptComments(source) { + return source.replace( + javaScriptStringOrCommentPattern, + (token, stringLiteral) => stringLiteral?.startsWith("`") + ? maskTemplateExpressionComments(stringLiteral) + : stringLiteral ?? token.replace(/[^\r\n]/g, " "), + ); +} + +function maskTemplateExpressionComments(template) { + let depth = 0; + return template.replace( + javaScriptTemplateTokenPattern, + (token, stringLiteral, brace, comment) => { + if (stringLiteral != null) return token; + if (brace != null) { + if (brace === "${" || (brace === "{" && depth > 0)) depth += 1; + else if (brace === "}" && depth > 0) depth -= 1; + return token; + } + return depth > 0 ? comment.replace(/[^\r\n]/g, " ") : token; + }, + ); +} + +function maskHashComments(source, extension) { + const masked = source.split(""); + const escape = extension === ".ps1" ? "`" : "\\"; + const outsideEscape = extension === ".ps1" || extension === ".sh"; + const doubledQuoteEscape = extension === ".ps1"; + let quote = null; + + for (let index = 0; index < source.length; index += 1) { + if (quote != null) { + if (source.startsWith(quote, index)) { + if ( + quote.length === 1 + && doubledQuoteEscape + && source[index + 1] === quote + ) { + index += 1; + } else { + index += quote.length - 1; + quote = null; + } + } else if ( + !quote.startsWith("'") + && source[index] === escape + && index + 1 < source.length + ) { + index += 1; + } + continue; + } + + if (outsideEscape && source[index] === escape && index + 1 < source.length) { + index += 1; + continue; + } + if (source[index] === "\"" || source[index] === "'") { + const delimiter = source[index]; + quote = (extension === ".py" || extension === ".toml") + && source.startsWith(delimiter.repeat(3), index) + ? delimiter.repeat(3) + : delimiter; + index += quote.length - 1; + continue; + } + if (source[index] !== "#") { + continue; + } + if ( + extension === ".sh" + && index > 0 + && !/[\s;&|()]/.test(source[index - 1]) + ) { + continue; + } + while (index < source.length && source[index] !== "\r" && source[index] !== "\n") { + masked[index] = " "; + index += 1; + } + index -= 1; + } + return masked.join(""); +} + +function maskYamlComments(source) { + const masked = source.split(""); + const blockScalarKinds = yamlBlockScalarKinds(source.split(/\r?\n/)); + let quote = null; + let quoteInBlockScalar = false; + let line = 0; + + for (let index = 0; index < source.length; index += 1) { + const current = source[index]; + if (index > 0 && source[index - 1] === "\n") line += 1; + if (quote != null) { + if ( + !quoteInBlockScalar + && quote === "'" + && current === "'" + && source[index + 1] === "'" + ) index += 1; + else if (quote === '"' && current === "\\") index += 1; + else if (current === quote) { + quote = null; + quoteInBlockScalar = false; + } + continue; + } + quoteInBlockScalar = blockScalarKinds[line] != null; + if ( + (current === "'" || current === '"') + && (quoteInBlockScalar || yamlQuoteStartsScalar(source, index)) + ) { + quote = current; + continue; + } + quoteInBlockScalar = false; + if ( + current !== "#" + || (index > 0 && source[index - 1] !== "\n" && !/[\t ]/.test(source[index - 1])) + ) { + continue; + } + while (index < source.length && source[index] !== "\r" && source[index] !== "\n") { + masked[index] = " "; + index += 1; + } + index -= 1; + } + return masked.join(""); +} + +function yamlQuoteStartsScalar(source, index) { + const lineStart = source.lastIndexOf("\n", index - 1) + 1; + const before = source.slice(lineStart, index); + const token = before.trimEnd(); + const delimiter = token.at(-1); + if (delimiter == null || ":[{,?".includes(delimiter)) return true; + return delimiter === "-" && /(?:^|[\t [{,])-$/.test(token); +} + +function logicalNonRustLines(lines, extension) { + const yamlScalarKinds = extension === ".yaml" || extension === ".yml" + ? yamlBlockScalarKinds(lines) + : null; + const logicalLines = []; + for (let start = 0; start < lines.length; start += 1) { + let end = start; + let text = lines[end]; + while ( + end + 1 < lines.length + && lineContinuationMarker(text, extension, yamlScalarKinds?.[end]) != null + ) { + text = yamlScalarKinds?.[end]?.endsWith(">") + ? `${text} ${lines[end + 1].trimStart()}` + : text.slice(0, -1) + lines[end + 1].trimStart(); + end += 1; + } + const yamlRunCommand = yamlRunCommandText(lines[start], extension); + logicalLines.push({ + text: yamlRunCommand ?? text, + startLine: start + 1, + endLine: end + 1, + shellLike: extension === ".sh" + || yamlScalarKinds?.[start]?.startsWith("run") + || isYamlRunLine(lines[start], extension), + }); + start = end; + } + return logicalLines; +} + +function yamlBlockScalarKinds(lines) { + const scalarKinds = Array(lines.length).fill(null); + let blockIndent = null; + let scalarKind = null; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + const indent = line.match(/^\s*/)[0].length; + if (blockIndent != null) { + if (line.trim() === "" || indent > blockIndent) { + scalarKinds[index] = scalarKind; + continue; + } + blockIndent = null; + } + const header = line.trimStart().match( + /^([^#\n]+):\s*([|>])(?:[1-9][+-]?|[+-][1-9]?)?(?:\s+#.*)?\s*$/, + ); + if (header != null) { + blockIndent = indent; + const key = header[1].trim().replace(/^-\s*/, ""); + const shellPrefix = ["run", "'run'", '"run"'].includes(key) ? "run" : ""; + scalarKind = `${shellPrefix}${header[2]}`; + } + } + return scalarKinds; +} + +function isYamlRunLine(line, extension) { + return (extension === ".yaml" || extension === ".yml") + && yamlRunLineMatch(line) != null; +} + +function yamlRunLineMatch(line) { + return line.match(/^\s*(?:-\s*)?(?:run|'run'|"run")\s*:\s*(.*)$/); +} + +function yamlRunCommandText(line, extension) { + if (extension !== ".yaml" && extension !== ".yml") return null; + const match = yamlRunLineMatch(line); + if (match == null) return null; + const scalar = match[1].trim(); + if (/^[|>](?:[1-9][+-]?|[+-][1-9]?)?$/.test(scalar)) return null; + if (scalar.startsWith("'") && scalar.endsWith("'")) return scalar.slice(1, -1).replaceAll("''", "'"); + if (scalar.startsWith('"') && scalar.endsWith('"')) return scalar.slice(1, -1).replace(/\\(["\\])/g, "$1"); + return scalar; +} + +function lineContinuationMarker(line, extension, yamlScalarKind) { + let marker = null; + if (executableJavaScriptExtensions.has(extension) || extension === ".sh") { + marker = "\\"; + } else if (extension === ".ps1") { + marker = "`"; + } else if ((extension === ".yaml" || extension === ".yml") && yamlScalarKind != null) { + marker = "\\`".includes(line.at(-1)) ? line.at(-1) : null; + } + if (marker == null || !line.endsWith(marker)) { + return null; + } + const escape = extension === ".ps1" || marker === "`" ? "`" : "\\"; + const quote = openQuoteAtEnd(line.slice(0, -1), escape); + if (quote === "'") { + return null; + } + if (executableJavaScriptExtensions.has(extension) && quote == null) { + return null; + } + return marker; +} + +function openQuoteAtEnd(text, escape) { + let quote = null; + for (let index = 0; index < text.length; index += 1) { + if (quote == null) { + if (text[index] === "\"" || text[index] === "'" || text[index] === "`") { + quote = text[index]; + } + continue; + } + if (quote !== "'" && text[index] === escape) { + index += 1; + } else if (text[index] === quote) { + quote = null; + } + } + return quote; +} + +function scanCorpusHarnessDependencies(prepared) { + const hits = []; + for (const line of prepared.logicalLines) { + const normalizedLine = normalizeNativeSeparators(line.text, line.shellLike).trim(); + const hasProtectedDependency = corpusHarnessDependencyRegexes.some((pattern) => + regexOccurrences(pattern, normalizedLine).some((occurrence) => + !harnessDependencyAllowed(prepared, line, pattern, occurrence) + ) + ); + if (hasProtectedDependency) { + hits.push( + `${prepared.filePath}:${line.startLine}:${prepared.sourceLines[line.startLine - 1]}`, + ); + } + } + return hits; +} + +function regexOccurrences(pattern, source) { + const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`; + return [...source.matchAll(new RegExp(pattern.source, flags))].map((match) => ({ + index: match.index, + length: match[0].length, + })); +} + +function harnessDependencyAllowed(prepared, line, pattern, occurrence) { + return allowedHarnessReferences.some(({ relativePath, includes, use }) => + allowedReferenceFileMatches(prepared.filePath, relativePath) + && allowedReferenceUseMatches(prepared, use, line.startLine, line.endLine) + && regexOccurrences(pattern, normalizeNativeSeparators(use)).some((allowed) => + allowed.index === occurrence.index && allowed.length === occurrence.length + ) + && pattern.test(normalizeNativeSeparators(includes)) + ); +} + +function compactHarnessDependencyAllowed(prepared, literals, marker) { + const startLine = Math.min(...literals.map(({ line }) => line)); + const endLine = Math.max(...literals.map(({ line }) => line)); + const source = prepared.sourceLines.slice(startLine - 1, endLine).join("\n").trim(); + return allowedHarnessReferences.some(({ relativePath, includes, use }) => { + const includesCompact = compactProductionSource(normalizeNativeSeparators(includes)); + return allowedReferenceFileMatches(prepared.filePath, relativePath) + && source === use + && allowedReferenceUseMatches(prepared, use, startLine, endLine) + && literals.some(({ literal }) => + normalizeNativeSeparators(staticStringLiteralContent(literal)).includes(includes) + ) + && (marker.includes(includesCompact) || includesCompact.includes(marker)); + }); +} + +function allowedReferenceFileMatches(filePath, relativePath) { + const roots = usesDefaultNonRustScanRoots ? [repoRoot] : nonRustScanRoots; + return roots.some((root) => + path.resolve(filePath) === path.resolve(root, relativePath) + ); +} + +function allowedReferenceUseMatches(prepared, use, startLine, endLine) { + const matches = prepared.logicalLines.filter((line) => + normalizeNativeSeparators(line.text, line.shellLike).trim() === use + ); + return matches.length === 1 + && matches[0].startLine === startLine + && matches[0].endLine === endLine; +} + function scanProductionFile(prepared, patterns, combinedRe) { - const lines = prepared.lines; + const lines = prepared.logicalLines ?? prepared.lines.map((text, index) => ({ + text, + startLine: index + 1, + endLine: index + 1, + })); + const sourceLines = prepared.sourceLines ?? prepared.lines; const hitsByPattern = new Map(); - for (let index = 0; index < lines.length; index += 1) { - if (!combinedRe.test(lines[index])) { + for (const line of lines) { + const normalizedLine = normalizeNativeSeparators(line.text, line.shellLike); + if (!combinedRe.test(normalizedLine)) { continue; } for (const { pattern, re } of patterns) { - if (re.test(lines[index]) && !lineAllowedForPattern(pattern, lines[index])) { + const sourceLine = sourceLines[line.startLine - 1]; + if (re.test(normalizedLine) && !lineAllowedForPattern(pattern, sourceLine)) { if (!hitsByPattern.has(pattern)) { hitsByPattern.set(pattern, []); } - hitsByPattern.get(pattern).push(`${prepared.filePath}:${index + 1}:${lines[index]}`); + hitsByPattern.get(pattern).push(`${prepared.filePath}:${line.startLine}:${sourceLine}`); } } } @@ -1069,8 +1630,11 @@ function scanProductionStringLiterals(prepared, pattern, re) { const lines = prepared.lines; const hits = []; for (let index = 0; index < lines.length; index += 1) { - for (const literal of rustStringLiteralsOnLine(lines[index])) { - if (re.test(literal) && !lineAllowedForPattern(pattern, lines[index])) { + for (const literal of staticStringLiteralsOnLine(lines[index])) { + if ( + re.test(normalizeNativeSeparators(literal)) + && !lineAllowedForPattern(pattern, lines[index]) + ) { hits.push(`${prepared.filePath}:${index + 1}:${lines[index]}`); break; } @@ -1080,13 +1644,43 @@ function scanProductionStringLiterals(prepared, pattern, re) { } function compactProductionSource(text) { - return rustStringLiteralContent(text) + return staticStringLiteralContent(text) .replace(/["'`]/g, "") .replace(/[^a-zA-Z0-9]+/g, "") .toLowerCase(); } -function rustStringLiteralContent(literal) { +function normalizeNativeSeparators(text, shellLike = false) { + const normalized = shellLike ? normalizeShellLikeText(text) : text; + return normalized.replaceAll("\\", "/"); +} + +function normalizeShellLikeText(text) { + let normalized = ""; + let quote = null; + for (let index = 0; index < text.length; index += 1) { + const current = text[index]; + if (quote == null && (current === "'" || current === '"')) { + quote = current; + continue; + } + if (current === quote) { + quote = null; + continue; + } + const escaped = current === "\\" && index + 1 < text.length + && (quote == null || (quote === '"' && '$`"\\'.includes(text[index + 1]))); + if (escaped) { + normalized += text[index + 1]; + index += 1; + } else { + normalized += current; + } + } + return normalized; +} + +function staticStringLiteralContent(literal) { const raw = literal.match(/^b?r(#+)?"([\s\S]*)"(#*)$/); if (raw && (raw[1] ?? "") === raw[3]) { return raw[2]; @@ -1097,33 +1691,64 @@ function rustStringLiteralContent(literal) { return literal; } -function scanProductionCompactPatterns(prepared, marker) { - const production = prepared.production; +function scanProductionCompactPatterns( + prepared, + marker, + minimumLiteralCount = 1, + allowSurroundingText = false, + matchAllowed = () => false, +) { + const production = prepared.staticSource ?? prepared.production; const markerLower = marker.toLowerCase(); const hits = []; if (prepared.literals == null) { - prepared.literals = rustStringLiteralSpans(production); + prepared.literals = staticStringLiteralSpans(production); } const literals = prepared.literals; for (let start = 0; start < literals.length; start += 1) { let compact = ""; + const segments = []; for (let end = start; end < literals.length; end += 1) { if ( end > start && !literalJoinGapAllowsCompactScan( production.slice(literals[end - 1].endOffset, literals[end].startOffset), + prepared.extension, + literals[end - 1].literal, + literals[end].literal, ) ) { break; } - compact += compactProductionSource(literals[end].literal); - if (compact === markerLower) { + const latestBoundary = compact.length; + const literalCompact = compactProductionSource(literals[end].literal); + compact += literalCompact; + segments.push({ + ...literals[end], + compactStart: latestBoundary, + compactEnd: compact.length, + }); + const occurrences = end - start + 1 >= minimumLiteralCount + ? compactMarkerOccurrences( + compact, + markerLower, + allowSurroundingText, + minimumLiteralCount > 1 ? latestBoundary : null, + ) + : []; + const rejected = occurrences.some(({ index, length }) => { + const matchedLiterals = segments.filter((literal) => + literal.compactStart < index + length && literal.compactEnd > index + ); + return !matchAllowed(matchedLiterals, markerLower); + }); + if (rejected) { hits.push( compactPatternHit(prepared.filePath, literals[start].line, literals[end].line, marker), ); break; } - if (compact.length >= markerLower.length) { + if (!allowSurroundingText && compact.length >= markerLower.length) { break; } } @@ -1131,7 +1756,33 @@ function scanProductionCompactPatterns(prepared, marker) { return hits; } -function rustStringLiteralSpans(text) { +function compactMarkerOccurrences(compact, marker, allowSurroundingText, requiredBoundary) { + if (!allowSurroundingText) { + return compact === marker + && ( + requiredBoundary == null + || (requiredBoundary > 0 && marker.length > requiredBoundary) + ) + ? [{ index: 0, length: marker.length }] + : []; + } + const occurrences = []; + for ( + let offset = compact.indexOf(marker); + offset >= 0; + offset = compact.indexOf(marker, offset + 1) + ) { + if ( + requiredBoundary == null + || (offset < requiredBoundary && offset + marker.length > requiredBoundary) + ) { + occurrences.push({ index: offset, length: marker.length }); + } + } + return occurrences; +} + +function staticStringLiteralSpans(text) { const literals = []; const lineStarts = [0]; for (let index = 0; index < text.length; index += 1) { @@ -1140,7 +1791,7 @@ function rustStringLiteralSpans(text) { } } - const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')/g; + const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*'|`(?:\\[\s\S]|[^`\\])*`)/g; let match; while ((match = stringLiteral.exec(text)) != null) { literals.push({ @@ -1167,8 +1818,22 @@ function lineNumberAtOffset(lineStarts, offset) { return high + 1; } -function literalJoinGapAllowsCompactScan(gap) { - return /^[\s,]*$/.test(gap); +function literalJoinGapAllowsCompactScan(gap, extension, previousLiteral, nextLiteral) { + if ( + (extension === ".yaml" || extension === ".yml") + && gap === "" + && previousLiteral.startsWith("'") + && nextLiteral.startsWith("'") + ) { + return false; + } + const withoutContinuations = gap + .replace(/\\\r?\n/g, "") + .replace(/`\r?\n/g, ""); + const withoutJoinCalls = withoutContinuations + .replace(/\.(?:concat|join)\s*\(/g, "") + .replace(/\bpath\.(?:join|resolve)\s*\(/g, ""); + return /^[\s,+()[\].]*$/.test(withoutJoinCalls); } function compactPatternHit(filePath, startLine, endLine, marker) { @@ -1179,9 +1844,9 @@ function compactPatternHit(filePath, startLine, endLine, marker) { ); } -function rustStringLiteralsOnLine(line) { +function staticStringLiteralsOnLine(line) { const literals = []; - const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')/g; + const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)/g; let match; while ((match = stringLiteral.exec(line)) != null) { literals.push(match[0]); @@ -1284,7 +1949,7 @@ for (const filePath of [...scanFiles].sort()) { } } -const corpusRegexPatterns = evalCorpusBoundaryPatterns().map((pattern) => ({ +const corpusRegexPatterns = evalCorpusBoundaryPatternList.map((pattern) => ({ pattern, re: new RegExp(pattern, "i"), })); @@ -1307,8 +1972,8 @@ for (const filePath of [...structuralFiles].sort()) { failed = true; } } - for (const pattern of evalCorpusCompactPatterns()) { - const hits = scanProductionCompactPatterns(prepared, pattern); + for (const pattern of evalCorpusCompactPatternList) { + const hits = scanProductionCompactPatterns(prepared, pattern, 1, true); if (hits.length > 0) { console.error(`Constructed production dependency on eval/query corpus /${pattern}/ in ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`); failed = true; @@ -1316,13 +1981,78 @@ for (const filePath of [...structuralFiles].sort()) { } } +const protectedNonRustScanFiles = new Set(); +for (const root of nonRustScanRoots) { + for (const filePath of walkProtectedNonRustFiles(root)) { + protectedNonRustScanFiles.add(filePath); + } +} +if (usesDefaultNonRustScanRoots) { + for (const filePath of requiredProtectedNonRustFiles) { + protectedNonRustScanFiles.add(filePath); + } +} +if (protectedNonRustScanFiles.size === 0) { + console.error("lint-retrieval-generalization: no protected non-Rust files found"); + process.exit(2); +} + +for (const filePath of [...protectedNonRustScanFiles].sort()) { + const prepared = prepareNonRustFile(filePath); + const harnessDependencyHits = scanCorpusHarnessDependencies(prepared); + if (harnessDependencyHits.length > 0) { + console.error( + `Protected non-Rust path depends on an evaluation/proof harness ${path.relative(repoRoot, filePath)}:\n${harnessDependencyHits.join("\n")}\n`, + ); + failed = true; + } + const productionHits = scanProductionFile( + prepared, + corpusRegexPatterns, + corpusCombinedRegex, + ); + for (const { pattern } of corpusRegexPatterns) { + const hits = productionHits.get(pattern) ?? []; + if (hits.length > 0) { + console.error( + `Banned eval/query pattern /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, + ); + failed = true; + } + } + for (const pattern of evalCorpusCompactPatternList) { + const hits = scanProductionCompactPatterns(prepared, pattern, 1, true); + if (hits.length > 0) { + console.error( + `Constructed eval/query dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, + ); + failed = true; + } + } + for (const pattern of corpusHarnessCompactPatternList) { + const hits = scanProductionCompactPatterns( + prepared, + pattern, + 2, + true, + (literals, marker) => compactHarnessDependencyAllowed(prepared, literals, marker), + ); + if (hits.length > 0) { + console.error( + `Constructed evaluation/proof harness dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, + ); + failed = true; + } + } +} + if (failed) { console.error( - "retrieval generalization lint failed: remove repo-specific literals from retrieval integration code", + "retrieval generalization lint failed: remove eval/query dependencies from protected product paths", ); process.exit(1); } console.log( - `lint-retrieval-generalization: ok (${scanDirs.length} retrieval dir(s), ${scanFiles.size} retrieval file(s), ${structuralFiles.size} production file(s), ${bannedPatterns.length} patterns)`, + `lint-retrieval-generalization: ok (${scanDirs.length} retrieval dir(s), ${scanFiles.size} retrieval file(s), ${structuralFiles.size} production file(s), ${protectedNonRustScanFiles.size} protected non-Rust file(s), ${bannedPatterns.length} patterns)`, );