From e6dd6e2a2cd0ed1251eb4facdc8c7507891d5aef Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 10:19:46 -0400 Subject: [PATCH 01/11] guard non-rust evidence boundaries --- .github/workflows/retrieval-sidecar-smoke.yml | 2 + CHANGELOG.md | 6 + .../tests/retrieval_generalization_guard.rs | 57 +++++ docs/testing/codestory-e2e-stats-log.md | 3 + docs/testing/performance-review-playbook.md | 37 ++- docs/testing/retrieval-architecture.md | 2 +- scripts/codestory-agent-ab-benchmark.mjs | 151 +---------- scripts/codestory-evidence-provenance.mjs | 144 +++++++++++ scripts/codestory-release-evidence-gate.mjs | 2 +- scripts/lint-retrieval-generalization.mjs | 234 ++++++++++++++++-- 10 files changed, 459 insertions(+), 179 deletions(-) create mode 100644 scripts/codestory-evidence-provenance.mjs diff --git a/.github/workflows/retrieval-sidecar-smoke.yml b/.github/workflows/retrieval-sidecar-smoke.yml index b0195a0d..7586f38f 100644 --- a/.github/workflows/retrieval-sidecar-smoke.yml +++ b/.github/workflows/retrieval-sidecar-smoke.yml @@ -24,6 +24,7 @@ on: - crates/codestory-indexer/Cargo.toml - crates/codestory-indexer/src/lib.rs - scripts/lint-retrieval-generalization.mjs + - scripts/codestory-evidence-provenance.mjs - scripts/codestory-release-evidence-gate.mjs - scripts/tests/codestory-release-evidence-gate.test.mjs - benchmarks/release-evidence/** @@ -57,6 +58,7 @@ on: - crates/codestory-indexer/Cargo.toml - crates/codestory-indexer/src/lib.rs - scripts/lint-retrieval-generalization.mjs + - scripts/codestory-evidence-provenance.mjs - scripts/codestory-release-evidence-gate.mjs - scripts/tests/codestory-release-evidence-gate.test.mjs - benchmarks/release-evidence/** diff --git a/CHANGELOG.md b/CHANGELOG.md index b72089d9..6ae9f00d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ project/workspace identity differs from the selected repository. `cache identity` now exposes the lossless project, workspace, artifact, and safe legacy-alias disposition used by that verification. +- Separated release-evidence provenance validation from the benchmark corpus + loader and expanded the generalization guard across the repository-controlled + non-Rust product and release-control boundary. Direct and adjacent/split + evaluation-corpus dependencies now fail in plugin launch/setup surfaces, + worktree/install tooling, runtime configuration, release scripts/workflows, + and the release evaluator; missing protected paths fail closed. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 7ad95b11..7cdd3317 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -131,6 +131,37 @@ fn run_lint_with_prompt_script_fixture(contents: &str) -> Output { .expect("run lint with prompt script fixture") } +fn run_lint_with_non_rust_fixture(contents: &str) -> Output { + let repo_root = workspace_root(); + let script = lint_script(&repo_root); + let fixture_root = TempDir::new().expect("create fixture root"); + std::fs::write( + fixture_root.path().join("neutral.rs"), + "pub fn repository_neutral_fixture() {}\n", + ) + .expect("write neutral Rust fixture"); + std::fs::write(fixture_root.path().join("leaked.mjs"), contents) + .expect("write non-Rust fixture"); + + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("lock lint script subprocess"); + Command::new("node") + .arg(&script) + .current_dir(&repo_root) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", + fixture_root.path(), + ) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_NON_RUST_SCAN_ROOTS", + fixture_root.path(), + ) + .output() + .expect("run lint with non-Rust fixture") +} + #[test] fn retrieval_generalization_lint_script_exits_clean_with_extra_fixture_root() { let repo_root = workspace_root(); @@ -386,6 +417,32 @@ pub const LEAKED_DEPENDENCY: &str = include_str!(concat!("../../benchmarks/", "t } } +#[test] +fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { + let output = run_lint_with_non_rust_fixture( + r#" +export const directCorpus = import( + "./fetch-holdout-repos.mjs" +); +export const splitCorpus = ["scripts/", "cross-repo-sourcetrail-queries.mjs"].join(""); +"#, + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "protected non-Rust corpus dependencies must fail lint; stderr={stderr}" + ); + for expected in [ + "fetch-holdout-repos.mjs", + "scriptscrossreposourcetrailqueriesmjs", + ] { + assert!( + stderr.to_ascii_lowercase().contains(expected), + "lint failure should identify non-Rust corpus dependency {expected}; stderr={stderr}" + ); + } +} + #[test] fn linter_fails_closed_when_one_prompt_corpus_entry_is_not_a_literal() { let output = run_lint_with_prompt_script_fixture( diff --git a/docs/testing/codestory-e2e-stats-log.md b/docs/testing/codestory-e2e-stats-log.md index 9d1a12a7..eada96a2 100644 --- a/docs/testing/codestory-e2e-stats-log.md +++ b/docs/testing/codestory-e2e-stats-log.md @@ -114,6 +114,7 @@ Rows whose commit cell ends in `+wt` were run from the working tree based on tha | 2026-07-13 | 0c2d830d+wt | pass, final macOS support pre-PR full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded the latest stats-log baseline by more than 25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1 because CODESTORY_REAL_REPO_DRILL_CASES was not set; checksum-matched native llama-server with explicit CPU allowance for the stats lane; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; symbol_search_docs 19,225; dense anchors 1,006; dense skips 19,225; retrieval_index_seconds 22.90; retrieval_status_seconds 1.03; repeat full refresh 15.00s with 0 embedded | 15.55 | 0.11 | 2.18 | 0.28 | 0.11 | 0.10 | 148,236 | 123,652 | 307 | 0 | 1,006 | true | | 2026-07-13 | a2e34238+wt | pass, PR #1048 review remediation full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded the latest stats-log baseline by more than 25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1 because CODESTORY_REAL_REPO_DRILL_CASES was not set; checksum-matched native llama-server with explicit CPU allowance for the stats lane; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; symbol_search_docs 19,246; dense anchors 1,006; dense skips 19,246; retrieval_index_seconds 22.02; retrieval_status_seconds 0.96; repeat full refresh 15.12s with 0 embedded | 15.94 | 0.10 | 2.11 | 0.24 | 0.09 | 0.08 | 148,383 | 123,783 | 307 | 0 | 1,006 | true | | 2026-07-13 | 72c674e5+1050wt | pass, issue #1050 same-timestamp content-fingerprint freshness full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded the latest stats-log baseline by more than 25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1 because CODESTORY_REAL_REPO_DRILL_CASES was not set; checksum-matched native llama-server with explicit CPU allowance for the stats lane; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; symbol_search_docs 19,251; dense anchors 1,007; dense skips 19,251; retrieval_index_seconds 31.65; retrieval_status_seconds 1.04; repeat full refresh 17.86s with 0 embedded | 19.72 | 0.18 | 2.44 | 0.31 | 0.18 | 0.14 | 148,437 | 123,824 | 307 | 0 | 1,007 | true | +| 2026-07-13 | 9c97808a+1054wt | pass, issue #1054 non-Rust corpus-boundary guard full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded the latest stats-log baseline by more than 25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1 because CODESTORY_REAL_REPO_DRILL_CASES was not set; checksum-matched native llama-server with explicit CPU allowance for the stats lane; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; symbol_search_docs 19,257; dense anchors 1,007; dense skips 19,257; retrieval_index_seconds 66.14; retrieval_status_seconds 0.91; repeat full refresh 16.78s with 0 embedded | 17.58 | 0.14 | 2.23 | 0.27 | 0.12 | 0.11 | 148,577 | 123,886 | 308 | 0 | 1,007 | true | ## Repeat And Report Timing @@ -164,6 +165,7 @@ gates. | 2026-07-13 | 0c2d830d+wt | final macOS support pre-PR full-sidecar stats on Apple Silicon; proof_tier full_sidecar; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; repeat graph 7.75s; repeat semantic 0.42s; repeat cache/search projection/index 0.00s/0.20s/1.86s | 15.00 | 1.79 | 0.78 | 1.02 | | 2026-07-13 | a2e34238+wt | PR #1048 review remediation full-sidecar stats on Apple Silicon; proof_tier full_sidecar; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; repeat graph 7.89s; repeat semantic 0.48s; repeat cache/search projection/index 0.00s/0.19s/1.84s | 15.12 | 1.68 | 0.73 | 0.95 | | 2026-07-13 | 72c674e5+1050wt | issue #1050 same-timestamp content-fingerprint freshness full-sidecar stats on Apple Silicon; proof_tier full_sidecar; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; repeat graph 8.89s; repeat semantic 0.62s; repeat cache/search projection/index 0.00s/0.23s/1.84s | 17.86 | 2.23 | 0.97 | 1.26 | +| 2026-07-13 | 9c97808a+1054wt | issue #1054 non-Rust corpus-boundary guard full-sidecar stats on Apple Silicon; proof_tier full_sidecar; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; repeat graph 8.64s; repeat semantic 0.53s; repeat cache/search projection/index 0.00s/0.23s/1.82s | 16.78 | 1.90 | 0.82 | 1.08 | ## Phase Metrics @@ -271,3 +273,4 @@ from this phase table rather than backfilled. | 2026-07-13 | a2e34238+wt | PR #1048 review remediation full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 22.02; retrieval_status_seconds 0.96; repeat full refresh 15.12s with 0 embedded; report_seconds 1.68; symbol_search_docs 19,246; dense anchors 1,006; reasons public_api 934, entrypoint 12, central_graph_node 48, component_report 12 | 15.94 | 7.85 | 0.50 | 0 | 0 | 0 | | 2026-07-13 | 72c674e5+1050wt | issue #1050 same-timestamp content-fingerprint freshness full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 31.65; retrieval_status_seconds 1.04; repeat full refresh 17.86s with 0 embedded; report_seconds 2.23; symbol_search_docs 19,251; dense anchors 1,007; reasons public_api 935, entrypoint 12, central_graph_node 48, component_report 12 | 19.72 | 10.28 | 0.59 | 0 | 0 | 0 | | 2026-07-13 | 72c674e5+1035wt | issue #1035 request-scoped filesystem-aware plugin routing full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warnings index_seconds and semantic_phase_seconds exceeded the latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 60.11; retrieval_status_seconds 0.82; repeat full refresh 17.20s with 0 embedded; report_seconds 1.84; symbol_search_docs 19,258; dense anchors 1,006; post-rebase focused identity tests and workspace check passed on 9c97808a | 20.02 | 0.12 | 2.19 | 0.26 | 0.10 | 0.09 | 148,419 | 123,802 | 307 | 0 | 1,006 | true | +| 2026-07-13 | 9c97808a+1054wt | issue #1054 non-Rust corpus-boundary guard full-sidecar stats on Apple Silicon; proof_tier full_sidecar; warning retrieval_index_seconds exceeded latest baseline by >25%; real drill intentionally skipped with CODESTORY_ALLOW_SKIP_REAL_REPO_DRILL_CASES=1; explicit CPU allowance; sidecar_status_after_retrieval_index full; search.sidecar_shadow_retrieval_mode full; retrieval_index_seconds 66.14; retrieval_status_seconds 0.91; repeat full refresh 16.78s with 0 embedded; report_seconds 1.90; symbol_search_docs 19,257; dense anchors 1,007; reasons public_api 935, entrypoint 12, central_graph_node 48, component_report 12 | 17.58 | 8.35 | 0.65 | 0 | 0 | 0 | diff --git a/docs/testing/performance-review-playbook.md b/docs/testing/performance-review-playbook.md index c9417a9e..42ff9789 100644 --- a/docs/testing/performance-review-playbook.md +++ b/docs/testing/performance-review-playbook.md @@ -164,12 +164,37 @@ ordinary CI but are explicitly `release_eligible: false`. A product profile must be created from provisioned raw evidence and explicitly approved before a release workflow can adopt this gate. -The corpus boundary is deliberately precise: it scans Rust production files in -all non-benchmark crates for direct path strings and adjacent/split literal -construction such as `concat!` and `include_str!`. It does not prove arbitrary -runtime path generation, non-Rust production surfaces, environment/config -indirection, or dependencies hidden behind external processes. Those surfaces -remain explicit follow-up audit scope rather than being claimed as covered. +The corpus boundary is deliberately precise. The generalization lint scans Rust +production files in all non-benchmark crates for copied corpus content, direct +paths, and adjacent/split literal construction. It also scans the following +repository-controlled non-Rust surfaces for direct and adjacent/split +dependencies on every inventoried evaluation/query corpus: + +The inventory is executable rather than documentation-only. Supported text and +configuration files under `scripts/`, `.github/scripts/`, +`.github/workflows/`, the shipped plugin, retrieval Compose configuration, and +native backend metadata enter the protected scan by default. Only the explicit +corpus/proof harness list and test/fixture directories are excluded; missing +required protected or harness paths fail the lint so moves require review. + +| Classification | Protected or allowed surface | Static contract | +| --- | --- | --- | +| Product launch and setup | Plugin manifests, hooks, MCP launcher, grounding skill guidance/setup scripts, Codex worktree setup, and the Windows installer | Protected; the configured directories are scanned recursively, while test and fixture directories are excluded. | +| Runtime configuration | Retrieval Compose configuration and native sidecar backend metadata | Protected; new supported text/config files under those directories enter the scan automatically. | +| Release control | Release/auto-release workflows, version detection/checking, package assembly, the release-evidence evaluator, and shared provenance validation | Protected; required files must exist, and protected non-Rust modules cannot import an explicitly classified corpus/proof harness module. | +| Explicit corpus harnesses | Task manifests, packet/repo benchmark drivers and scorers, holdout provisioning, release-candidate measurement, the retrieval-sidecar contract workflow, and test fixtures | Allowed to load evaluation corpora; these paths are evidence producers, not product or release-decision logic. Other scripts and workflows remain protected by default. | +| Environment and generated inputs | Environment values, downloaded manifests/models, generated evidence, and workflow artifacts | Static code paths are protected where listed above. Runtime values and generated bytes require the existing schema, hash, identity, and provenance checks. | +| External processes | Git, Docker/Qdrant, embedding servers, and agent executables | Static command construction is protected where it is part of a listed surface. The external executable's internal behavior is outside this repository scan. | + +The release evaluator now imports repository/cache provenance checks from +`scripts/codestory-evidence-provenance.mjs`, a corpus-neutral module shared with +the benchmark harness. It no longer loads the packet benchmark module—and its +query catalog—as a transitive release-control dependency. + +This guard does not prove arbitrary runtime string generation, environment +values, generated content, or external-process behavior. Those boundaries stay +fail closed through the release-evidence attestation/runtime checks where a +machine-verifiable contract exists; otherwise they remain explicit non-claims. Do not promote importable or rebuildable graph/sidecar artifacts in this slice. A follow-up PR for that idea must require provenance before reuse: source root, diff --git a/docs/testing/retrieval-architecture.md b/docs/testing/retrieval-architecture.md index 83094f99..cafce909 100644 --- a/docs/testing/retrieval-architecture.md +++ b/docs/testing/retrieval-architecture.md @@ -158,7 +158,7 @@ Non-claims: | CLI lifecycle | `codestory-cli` `retrieval up\|down\|status\|index\|query` | Local data dirs, health JSON, standalone query | | Packet integration | `codestory-runtime/src/agent/retrieval_primary.rs` | Primary sidecar path, diagnostic traces, promotion warnings | | Nucleo policy | `codestory-runtime/src/agent/nucleo_policy.rs` | Suppresses Nucleo O(n) scan on sidecar primary; disabled sidecars are not valid product evidence | -| Generalization lint | `scripts/lint-retrieval-generalization.mjs` | Derives banned identities, prompts, claims, paths, and query/probe phrases from benchmark manifests, script prompt/query catalogs, and the eval-only probe manifest/source, then scans Rust production retrieval trees after masking test-only items (CI via Rust guard test); missing, malformed, or partially parsed corpora fail closed | +| Generalization lint | `scripts/lint-retrieval-generalization.mjs` | Derives banned identities, prompts, claims, paths, and query/probe phrases from benchmark manifests, script prompt/query catalogs, and the eval-only probe manifest/source. It scans Rust production retrieval trees after masking test-only items and structurally scans the inventoried non-Rust product/release-control boundary for direct or adjacent/split corpus dependencies (CI via the Rust guard test); missing protected paths and missing, malformed, or partially parsed corpora fail closed. | All planned retrieval stages use the same fixed-capacity worker pool, including symbol-like and natural-language queries. Each job carries the request deadline diff --git a/scripts/codestory-agent-ab-benchmark.mjs b/scripts/codestory-agent-ab-benchmark.mjs index df100778..bebfa279 100644 --- a/scripts/codestory-agent-ab-benchmark.mjs +++ b/scripts/codestory-agent-ab-benchmark.mjs @@ -22,6 +22,12 @@ import { shouldPrepareRetrievalIndex, unsupportedSidecarContractRequests, } from "./codestory-benchmark-contract.mjs"; +import { + cacheProvenanceBlockers, + isImmutableCommitRef, + isTrustedPublishableRepoUrl, + repoProvenanceBlockers, +} from "./codestory-evidence-provenance.mjs"; const scriptDir = path.dirname(fileURLToPath(import.meta.url)); const benchmarkHarnessPath = fileURLToPath(import.meta.url); @@ -1012,39 +1018,6 @@ function uniqueTaskRepos(tasks) { return repos; } -function isTrustedPublishableRepoUrl(url) { - try { - const parsed = new URL(String(url ?? "")); - if ( - parsed.protocol !== "https:" || - parsed.hostname.toLowerCase() !== "github.com" || - parsed.username || - parsed.password || - parsed.search || - parsed.hash - ) { - return false; - } - const parts = parsed.pathname.split("/").filter(Boolean); - return ( - parts.length === 2 && - /^[A-Za-z0-9_.-]+$/.test(parts[0]) && - /^[A-Za-z0-9_.-]+(?:\.git)?$/.test(parts[1]) - ); - } catch { - return false; - } -} - -function normalizeTrustedPublishableRepoUrl(url) { - if (!isTrustedPublishableRepoUrl(url)) { - return null; - } - const parsed = new URL(String(url)); - const [owner, repo] = parsed.pathname.split("/").filter(Boolean); - return `${owner.toLowerCase()}/${repo.replace(/\.git$/i, "").toLowerCase()}`; -} - function manifestRepoMaterializationBlockers(tasks, opts = {}) { if (!opts.publishable || !opts.materializeRepos) { return []; @@ -4746,118 +4719,6 @@ function packetRuntimeMarkdownRow(row) { return `| ${cells.join(" | ")} |`; } -function repoProvenanceBlockers(result) { - const provenance = result.repo_provenance; - if (!provenance) { - return ["missing repo provenance"]; - } - const reasons = []; - if (provenance.manifest_overridden_by_builtin) { - reasons.push("manifest repo was overridden by a built-in checkout"); - } - const configuredRef = provenance.configured?.ref ?? null; - const manifestRef = provenance.manifest?.ref ?? null; - const configuredCommit = normalizeImmutableCommitRef(configuredRef); - const manifestCommit = manifestRef ? normalizeImmutableCommitRef(manifestRef) : null; - const gitHead = normalizeImmutableCommitRef(provenance.git_head); - if (!configuredCommit) { - reasons.push("repo ref is not pinned to a full immutable commit SHA"); - } - if (manifestRef && configuredRef && manifestCommit !== configuredCommit) { - reasons.push(`manifest ref ${manifestRef} does not match configured ref ${configuredRef}`); - } - if (!gitHead) { - reasons.push("missing git head"); - } else if (configuredCommit && gitHead !== configuredCommit) { - reasons.push(`git head ${provenance.git_head} does not match configured ref ${configuredRef}`); - } - const configuredUrl = provenance.configured?.url ?? null; - const manifestUrl = provenance.manifest?.url ?? null; - const gitOrigin = provenance.git_origin ?? null; - const configuredRepo = normalizeTrustedPublishableRepoUrl(configuredUrl); - const manifestRepo = manifestUrl ? normalizeTrustedPublishableRepoUrl(manifestUrl) : null; - const originRepo = gitOrigin ? normalizeTrustedPublishableRepoUrl(gitOrigin) : null; - if (!configuredRepo) { - reasons.push("configured repo URL is not a trusted GitHub HTTPS repo URL"); - } - if (!manifestUrl) { - reasons.push("missing manifest repo URL"); - } else if (!manifestRepo) { - reasons.push("manifest repo URL is not a trusted GitHub HTTPS repo URL"); - } - if (configuredRepo && manifestUrl && manifestRepo && manifestRepo !== configuredRepo) { - reasons.push(`manifest repo URL ${manifestUrl} does not match configured URL ${configuredUrl}`); - } - if (!originRepo) { - reasons.push("git origin is missing or is not a trusted GitHub HTTPS repo URL"); - } else if (configuredRepo && originRepo !== configuredRepo) { - reasons.push(`git origin ${gitOrigin} does not match configured URL ${configuredUrl}`); - } - if (provenance.git_dirty !== false) { - reasons.push(provenance.git_dirty ? "repo checkout is dirty" : "repo cleanliness is unknown"); - } - return reasons; -} - -function isImmutableCommitRef(ref) { - return /^[0-9a-f]{40}$/i.test(String(ref ?? "").trim()); -} - -function normalizeImmutableCommitRef(ref) { - const value = String(ref ?? "").trim(); - return isImmutableCommitRef(value) ? value.toLowerCase() : null; -} - -function cacheProvenanceBlockers(result) { - const provenance = result.codestory_cache_provenance; - if (!provenance) { - return ["missing CodeStory cache provenance"]; - } - const reasons = []; - if (provenance.doctor_status !== "pass") { - reasons.push("CodeStory doctor provenance failed"); - } - if (!provenance.storage_path) { - reasons.push("missing CodeStory cache path"); - } - if (!provenance.cache_policy) { - reasons.push("missing CodeStory cache policy"); - } - if (provenance.cache_policy === "unprepared-cache-blocked") { - reasons.push("CodeStory sidecar cache was not prepared"); - } - if (provenance.retrieval_mode !== "full") { - reasons.push(`CodeStory retrieval mode=${provenance.retrieval_mode ?? "unknown"}; expected full`); - } - if (!provenance.sidecar_generation) { - reasons.push("missing CodeStory sidecar generation"); - } - if (provenance.manifest_embedding_backend !== "llamacpp:bge-base-en-v1.5") { - reasons.push( - `CodeStory sidecar embedding backend=${provenance.manifest_embedding_backend ?? "unknown"}; expected llamacpp:bge-base-en-v1.5`, - ); - } - if (provenance.semantic_backend == null) { - reasons.push("missing CodeStory semantic backend"); - } - if (provenance.local_only !== true) { - reasons.push(`CodeStory local-only guarantee is not proven (${provenance.locality_kind ?? "unknown"})`); - } - if (provenance.indexed !== true) { - reasons.push("CodeStory cache is not indexed"); - } - if (provenance.freshness_status !== "fresh") { - reasons.push(`CodeStory cache freshness=${provenance.freshness_status ?? "unknown"}`); - } - if (provenance.semantic_ready !== true) { - reasons.push("CodeStory semantic docs are not ready"); - } - if (provenance.indexing_in_timed_run == null) { - reasons.push("missing timed-run indexing provenance"); - } - return reasons; -} - function qualityFailureReasons(quality) { if (!quality) { return ["missing_quality_score"]; diff --git a/scripts/codestory-evidence-provenance.mjs b/scripts/codestory-evidence-provenance.mjs new file mode 100644 index 00000000..c640c3ba --- /dev/null +++ b/scripts/codestory-evidence-provenance.mjs @@ -0,0 +1,144 @@ +export function isTrustedPublishableRepoUrl(url) { + try { + const parsed = new URL(String(url ?? "")); + if ( + parsed.protocol !== "https:" + || parsed.hostname.toLowerCase() !== "github.com" + || parsed.username + || parsed.password + || parsed.search + || parsed.hash + ) { + return false; + } + const parts = parsed.pathname.split("/").filter(Boolean); + return ( + parts.length === 2 + && /^[A-Za-z0-9_.-]+$/.test(parts[0]) + && /^[A-Za-z0-9_.-]+(?:\.git)?$/.test(parts[1]) + ); + } catch { + return false; + } +} + +function normalizeTrustedPublishableRepoUrl(url) { + if (!isTrustedPublishableRepoUrl(url)) { + return null; + } + const parsed = new URL(String(url)); + const [owner, repo] = parsed.pathname.split("/").filter(Boolean); + return `${owner.toLowerCase()}/${repo.replace(/\.git$/i, "").toLowerCase()}`; +} + +export function isImmutableCommitRef(ref) { + return /^[0-9a-f]{40}$/i.test(String(ref ?? "").trim()); +} + +function normalizeImmutableCommitRef(ref) { + const value = String(ref ?? "").trim(); + return isImmutableCommitRef(value) ? value.toLowerCase() : null; +} + +export function repoProvenanceBlockers(result) { + const provenance = result.repo_provenance; + if (!provenance) { + return ["missing repo provenance"]; + } + const reasons = []; + if (provenance.manifest_overridden_by_builtin) { + reasons.push("manifest repo was overridden by a built-in checkout"); + } + const configuredRef = provenance.configured?.ref ?? null; + const manifestRef = provenance.manifest?.ref ?? null; + const configuredCommit = normalizeImmutableCommitRef(configuredRef); + const manifestCommit = manifestRef ? normalizeImmutableCommitRef(manifestRef) : null; + const gitHead = normalizeImmutableCommitRef(provenance.git_head); + if (!configuredCommit) { + reasons.push("repo ref is not pinned to a full immutable commit SHA"); + } + if (manifestRef && configuredRef && manifestCommit !== configuredCommit) { + reasons.push(`manifest ref ${manifestRef} does not match configured ref ${configuredRef}`); + } + if (!gitHead) { + reasons.push("missing git head"); + } else if (configuredCommit && gitHead !== configuredCommit) { + reasons.push(`git head ${provenance.git_head} does not match configured ref ${configuredRef}`); + } + const configuredUrl = provenance.configured?.url ?? null; + const manifestUrl = provenance.manifest?.url ?? null; + const gitOrigin = provenance.git_origin ?? null; + const configuredRepo = normalizeTrustedPublishableRepoUrl(configuredUrl); + const manifestRepo = manifestUrl ? normalizeTrustedPublishableRepoUrl(manifestUrl) : null; + const originRepo = gitOrigin ? normalizeTrustedPublishableRepoUrl(gitOrigin) : null; + if (!configuredRepo) { + reasons.push("configured repo URL is not a trusted GitHub HTTPS repo URL"); + } + if (!manifestUrl) { + reasons.push("missing manifest repo URL"); + } else if (!manifestRepo) { + reasons.push("manifest repo URL is not a trusted GitHub HTTPS repo URL"); + } + if (configuredRepo && manifestUrl && manifestRepo && manifestRepo !== configuredRepo) { + reasons.push(`manifest repo URL ${manifestUrl} does not match configured URL ${configuredUrl}`); + } + if (!originRepo) { + reasons.push("git origin is missing or is not a trusted GitHub HTTPS repo URL"); + } else if (configuredRepo && originRepo !== configuredRepo) { + reasons.push(`git origin ${gitOrigin} does not match configured URL ${configuredUrl}`); + } + if (provenance.git_dirty !== false) { + reasons.push(provenance.git_dirty ? "repo checkout is dirty" : "repo cleanliness is unknown"); + } + return reasons; +} + +export function cacheProvenanceBlockers(result) { + const provenance = result.codestory_cache_provenance; + if (!provenance) { + return ["missing CodeStory cache provenance"]; + } + const reasons = []; + if (provenance.doctor_status !== "pass") { + reasons.push("CodeStory doctor provenance failed"); + } + if (!provenance.storage_path) { + reasons.push("missing CodeStory cache path"); + } + if (!provenance.cache_policy) { + reasons.push("missing CodeStory cache policy"); + } + if (provenance.cache_policy === "unprepared-cache-blocked") { + reasons.push("CodeStory sidecar cache was not prepared"); + } + if (provenance.retrieval_mode !== "full") { + reasons.push(`CodeStory retrieval mode=${provenance.retrieval_mode ?? "unknown"}; expected full`); + } + if (!provenance.sidecar_generation) { + reasons.push("missing CodeStory sidecar generation"); + } + if (provenance.manifest_embedding_backend !== "llamacpp:bge-base-en-v1.5") { + reasons.push( + `CodeStory sidecar embedding backend=${provenance.manifest_embedding_backend ?? "unknown"}; expected llamacpp:bge-base-en-v1.5`, + ); + } + if (provenance.semantic_backend == null) { + reasons.push("missing CodeStory semantic backend"); + } + if (provenance.local_only !== true) { + reasons.push(`CodeStory local-only guarantee is not proven (${provenance.locality_kind ?? "unknown"})`); + } + if (provenance.indexed !== true) { + reasons.push("CodeStory cache is not indexed"); + } + if (provenance.freshness_status !== "fresh") { + reasons.push(`CodeStory cache freshness=${provenance.freshness_status ?? "unknown"}`); + } + if (provenance.semantic_ready !== true) { + reasons.push("CodeStory semantic docs are not ready"); + } + if (provenance.indexing_in_timed_run == null) { + reasons.push("missing timed-run indexing provenance"); + } + return reasons; +} diff --git a/scripts/codestory-release-evidence-gate.mjs b/scripts/codestory-release-evidence-gate.mjs index d3ceec3a..d9c89c52 100644 --- a/scripts/codestory-release-evidence-gate.mjs +++ b/scripts/codestory-release-evidence-gate.mjs @@ -9,7 +9,7 @@ import { pathToFileURL } from "node:url"; import { cacheProvenanceBlockers, repoProvenanceBlockers, -} from "./codestory-agent-ab-benchmark.mjs"; +} from "./codestory-evidence-provenance.mjs"; const METRICS = [ "status_seconds", "local_grounding_seconds", "convergence_seconds", diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 2ea3a2cc..c1706152 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -1,12 +1,11 @@ #!/usr/bin/env node /** - * CI guard: ban repo-specific path literals in retrieval integration production code. - * Scope is Rust production retrieval integration files. Benchmark/eval harness - * scripts and the env-gated eval probe module intentionally live outside this - * guard because their manifests name holdout repos; keep that boundary explicit - * instead of treating them as product code. - * Scans Rust files after masking `#[cfg(test)]` items/modules so test fixtures - * do not define the production contract. + * CI guard: keep production and release-control code independent from checked + * evaluation/query corpora. Rust production is checked for derived corpus + * content and structural paths after masking `#[cfg(test)]` items. Inventoried + * non-Rust product/release surfaces are checked for direct and adjacent/split + * corpus dependencies. Explicit benchmark/proof harnesses remain outside the + * protected scan because they must load those corpora. */ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; @@ -24,6 +23,85 @@ const explicitScanRoots = ( ) .split(path.delimiter) .filter(Boolean); +const explicitNonRustScanRoots = ( + process.env.CODESTORY_RETRIEVAL_GENERALIZATION_NON_RUST_SCAN_ROOTS ?? "" +) + .split(path.delimiter) + .filter(Boolean); + +const protectedNonRustDirs = [ + path.join(repoRoot, "scripts"), + path.join(repoRoot, ".github", "scripts"), + path.join(repoRoot, ".github", "workflows"), + path.join(repoRoot, "plugins", "codestory"), + path.join(repoRoot, "docker"), + path.join(repoRoot, "crates", "codestory-retrieval", "assets"), +]; + +const requiredProtectedNonRustFiles = [ + path.join(repoRoot, "scripts", "codestory-evidence-provenance.mjs"), + path.join(repoRoot, "scripts", "codestory-release-evidence-gate.mjs"), + path.join(repoRoot, "scripts", "codex-worktree-setup.mjs"), + path.join(repoRoot, "scripts", "codex-worktree-setup.ps1"), + path.join(repoRoot, "scripts", "codex-worktree-setup.sh"), + path.join(repoRoot, "scripts", "install-codestory.ps1"), + path.join(repoRoot, ".github", "scripts", "check-codestory-release.py"), + path.join(repoRoot, ".github", "scripts", "detect-codestory-release.py"), + path.join(repoRoot, ".github", "scripts", "package-codestory-release.py"), + path.join(repoRoot, ".github", "workflows", "auto-release.yml"), + path.join(repoRoot, ".github", "workflows", "release.yml"), +]; + +const corpusHarnessNonRustFiles = new Set([ + path.join(repoRoot, "scripts", "autoresearch-pipeline-score.mjs"), + path.join(repoRoot, "scripts", "codestory-agent-ab-benchmark.mjs"), + path.join(repoRoot, "scripts", "codestory-agent-ab-score.mjs"), + path.join(repoRoot, "scripts", "codestory-agent-value-score.mjs"), + path.join(repoRoot, "scripts", "codestory-benchmark-contract.mjs"), + path.join(repoRoot, "scripts", "codestory-language-holdout-integrity.mjs"), + path.join(repoRoot, "scripts", "codestory-manual-friction-check.mjs"), + path.join(repoRoot, "scripts", "cross-repo-sourcetrail-queries.mjs"), + path.join(repoRoot, "scripts", "fetch-holdout-repos.mjs"), + path.join(repoRoot, "scripts", "lint-retrieval-generalization.mjs"), + path.join(repoRoot, "scripts", "measure-peak-memory.ps1"), + path.join(repoRoot, "scripts", "prove-drill-packet-parity.mjs"), + path.join(repoRoot, "scripts", "score-drill-ledger.mjs"), + path.join(repoRoot, "scripts", "setup-retrieval-env.mjs"), + path.join(repoRoot, "scripts", "setup-retrieval-env.ps1"), + path.join(repoRoot, ".github", "workflows", "release-candidate-evidence.yml"), + path.join(repoRoot, ".github", "workflows", "retrieval-sidecar-smoke.yml"), +].map((filePath) => path.resolve(filePath))); +const corpusHarnessModuleNames = new Set( + [...corpusHarnessNonRustFiles] + .filter((filePath) => [".cjs", ".js", ".mjs"].includes(path.extname(filePath))) + .map((filePath) => path.basename(filePath)), +); + +const protectedNonRustExtensions = new Set([ + ".cjs", ".js", ".json", ".md", ".mjs", ".ps1", ".py", ".sh", ".toml", ".ts", + ".tsx", ".yaml", ".yml", +]); + +const defaultNonRustScanRoots = protectedNonRustDirs; +const usesDefaultNonRustScanRoots = explicitNonRustScanRoots.length === 0; +const nonRustScanRoots = usesDefaultNonRustScanRoots + ? defaultNonRustScanRoots + : explicitNonRustScanRoots.filter((root) => root && existsSync(root)); + +if (usesDefaultNonRustScanRoots) { + const missingProtectedPaths = [ + ...defaultNonRustScanRoots, + ...requiredProtectedNonRustFiles, + ...corpusHarnessNonRustFiles, + ].filter((requiredPath) => !existsSync(requiredPath)); + if (missingProtectedPaths.length > 0) { + console.error("lint-retrieval-generalization: missing protected non-Rust path(s)"); + for (const missingPath of missingProtectedPaths) { + console.error(` ${path.relative(repoRoot, missingPath)}`); + } + process.exit(2); + } +} const structuralScanDirs = readdirSync(path.join(repoRoot, "crates"), { withFileTypes: true }) .filter((entry) => entry.isDirectory() && entry.name !== "codestory-bench") @@ -117,6 +195,11 @@ if (scanDirs.length === 0 && productionOnlyFiles.length === 0) { process.exit(2); } +const evalCorpusBoundaryPatternList = evalCorpusBoundaryPatterns(); +const evalCorpusCompactPatternList = evalCorpusCompactPatterns( + evalCorpusBoundaryPatternList, +); + const bannedPatterns = [ "payload_config", "freelancer", @@ -206,7 +289,7 @@ const bannedPatterns = [ "nvm", "install\\.sh\\s+nvm", "bash_completion\\s+__nvm", - ...evalCorpusBoundaryPatterns(), + ...evalCorpusBoundaryPatternList, ...benchmarkManifestDerivedPatterns(), ...benchmarkEvalProbeDerivedPatterns(), ...benchmarkScriptPromptDerivedPatterns(), @@ -234,7 +317,7 @@ const bannedCompactPatterns = [ "datarequest", "sessiondelegate", "sourceanimatecss", - ...evalCorpusCompactPatterns(), + ...evalCorpusCompactPatternList, ]; const allowedPatternLines = [ @@ -262,11 +345,12 @@ function evalCorpusBoundaryPatterns() { return [ ...evalCorpusRoots.map((root) => path.relative(repoRoot, root).replaceAll(path.sep, "/")), ...corpusFiles.map((filePath) => path.relative(repoRoot, filePath).replaceAll(path.sep, "/")), + ...benchmarkIdentityScriptFiles.map((filePath) => path.basename(filePath)), ].map(escapeRegExp); } -function evalCorpusCompactPatterns() { - return evalCorpusBoundaryPatterns() +function evalCorpusCompactPatterns(boundaryPatterns) { + return boundaryPatterns .map((pattern) => compactProductionSource(pattern.replaceAll("\\", ""))) .filter((pattern) => pattern.length >= 12); } @@ -439,13 +523,13 @@ function addEvalProbeSourceQueryMarkers(markers, source) { let arrayMatch; while ((arrayMatch = queryArrayCall.exec(source)) != null) { const body = arrayMatch[1]; - const literals = rustStringLiteralSpans(body); + const literals = staticStringLiteralSpans(body); if (literals.length === 0 || rustArrayNonLiteralRemainder(body, literals).trim() !== "") { throw new Error("eval probe source query array contains an unparsed entry"); } for (const { literal } of literals) { parsedLiteralCount += 1; - addSpecificMarker(markers, rustStringLiteralContent(literal), { + addSpecificMarker(markers, staticStringLiteralContent(literal), { allowSpecificComposite: true, }); } @@ -558,6 +642,28 @@ function walkFiles(root, predicate) { return files; } +function walkProtectedNonRustFiles(root) { + return walkFiles(root, (filePath) => { + if (!protectedNonRustExtensions.has(path.extname(filePath).toLowerCase())) { + return false; + } + if (!usesDefaultNonRustScanRoots) { + return true; + } + if (corpusHarnessNonRustFiles.has(path.resolve(filePath))) { + return false; + } + const segments = path.relative(repoRoot, filePath).split(path.sep); + const baseName = path.basename(filePath); + return ( + !segments.includes("tests") + && !segments.includes("fixtures") + && !baseName.startsWith("test-") + && !baseName.includes(".test.") + ); + }); +} + function addSpecificMarker(markers, value, options = {}) { if (typeof value !== "string") { return; @@ -1046,6 +1152,34 @@ function prepareProductionFile(filePath) { }; } +function prepareNonRustFile(filePath) { + const production = readFileSync(filePath, "utf8"); + return { + filePath, + production, + lines: production.split(/\r?\n/), + literals: null, + }; +} + +function scanCorpusHarnessImports(prepared) { + const hits = []; + const importPattern = /(?:\bfrom\s+|\bimport\s*\(?\s*|\brequire\s*\(\s*)["'`]([^"'`]+)["'`]/g; + let match; + while ((match = importPattern.exec(prepared.production)) != null) { + const moduleName = path.posix.basename( + match[1].split(/[?#]/, 1)[0].replaceAll("\\", "/"), + ); + if (corpusHarnessModuleNames.has(moduleName)) { + const line = prepared.production.slice(0, match.index).split(/\r?\n/).length; + hits.push( + `${prepared.filePath}:${line}:${match[0].replace(/\s+/g, " ")}`, + ); + } + } + return hits; +} + function scanProductionFile(prepared, patterns, combinedRe) { const lines = prepared.lines; const hitsByPattern = new Map(); @@ -1069,7 +1203,7 @@ function scanProductionStringLiterals(prepared, pattern, re) { const lines = prepared.lines; const hits = []; for (let index = 0; index < lines.length; index += 1) { - for (const literal of rustStringLiteralsOnLine(lines[index])) { + for (const literal of staticStringLiteralsOnLine(lines[index])) { if (re.test(literal) && !lineAllowedForPattern(pattern, lines[index])) { hits.push(`${prepared.filePath}:${index + 1}:${lines[index]}`); break; @@ -1080,13 +1214,13 @@ function scanProductionStringLiterals(prepared, pattern, re) { } function compactProductionSource(text) { - return rustStringLiteralContent(text) + return staticStringLiteralContent(text) .replace(/["'`]/g, "") .replace(/[^a-zA-Z0-9]+/g, "") .toLowerCase(); } -function rustStringLiteralContent(literal) { +function staticStringLiteralContent(literal) { const raw = literal.match(/^b?r(#+)?"([\s\S]*)"(#*)$/); if (raw && (raw[1] ?? "") === raw[3]) { return raw[2]; @@ -1102,7 +1236,7 @@ function scanProductionCompactPatterns(prepared, marker) { const markerLower = marker.toLowerCase(); const hits = []; if (prepared.literals == null) { - prepared.literals = rustStringLiteralSpans(production); + prepared.literals = staticStringLiteralSpans(production); } const literals = prepared.literals; for (let start = 0; start < literals.length; start += 1) { @@ -1131,7 +1265,7 @@ function scanProductionCompactPatterns(prepared, marker) { return hits; } -function rustStringLiteralSpans(text) { +function staticStringLiteralSpans(text) { const literals = []; const lineStarts = [0]; for (let index = 0; index < text.length; index += 1) { @@ -1140,7 +1274,7 @@ function rustStringLiteralSpans(text) { } } - const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')/g; + const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)/g; let match; while ((match = stringLiteral.exec(text)) != null) { literals.push({ @@ -1168,7 +1302,10 @@ function lineNumberAtOffset(lineStarts, offset) { } function literalJoinGapAllowsCompactScan(gap) { - return /^[\s,]*$/.test(gap); + const withoutJoinCalls = gap + .replace(/\.(?:concat|join)\s*\(/g, "") + .replace(/\bpath\.(?:join|resolve)\s*\(/g, ""); + return /^[\s,+()[\].]*$/.test(withoutJoinCalls); } function compactPatternHit(filePath, startLine, endLine, marker) { @@ -1179,9 +1316,9 @@ function compactPatternHit(filePath, startLine, endLine, marker) { ); } -function rustStringLiteralsOnLine(line) { +function staticStringLiteralsOnLine(line) { const literals = []; - const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')/g; + const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)/g; let match; while ((match = stringLiteral.exec(line)) != null) { literals.push(match[0]); @@ -1284,7 +1421,7 @@ for (const filePath of [...scanFiles].sort()) { } } -const corpusRegexPatterns = evalCorpusBoundaryPatterns().map((pattern) => ({ +const corpusRegexPatterns = evalCorpusBoundaryPatternList.map((pattern) => ({ pattern, re: new RegExp(pattern, "i"), })); @@ -1307,7 +1444,7 @@ for (const filePath of [...structuralFiles].sort()) { failed = true; } } - for (const pattern of evalCorpusCompactPatterns()) { + for (const pattern of evalCorpusCompactPatternList) { const hits = scanProductionCompactPatterns(prepared, pattern); if (hits.length > 0) { console.error(`Constructed production dependency on eval/query corpus /${pattern}/ in ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`); @@ -1316,13 +1453,58 @@ for (const filePath of [...structuralFiles].sort()) { } } +const protectedNonRustScanFiles = new Set(); +for (const root of nonRustScanRoots) { + for (const filePath of walkProtectedNonRustFiles(root)) { + protectedNonRustScanFiles.add(filePath); + } +} +if (protectedNonRustScanFiles.size === 0) { + console.error("lint-retrieval-generalization: no protected non-Rust files found"); + process.exit(2); +} + +for (const filePath of [...protectedNonRustScanFiles].sort()) { + const prepared = prepareNonRustFile(filePath); + const harnessImportHits = scanCorpusHarnessImports(prepared); + if (harnessImportHits.length > 0) { + console.error( + `Protected non-Rust path imports an evaluation/proof harness ${path.relative(repoRoot, filePath)}:\n${harnessImportHits.join("\n")}\n`, + ); + failed = true; + } + const productionHits = scanProductionFile( + prepared, + corpusRegexPatterns, + corpusCombinedRegex, + ); + for (const { pattern } of corpusRegexPatterns) { + const hits = productionHits.get(pattern) ?? []; + if (hits.length > 0) { + console.error( + `Banned eval/query pattern /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, + ); + failed = true; + } + } + for (const pattern of evalCorpusCompactPatternList) { + const hits = scanProductionCompactPatterns(prepared, pattern); + if (hits.length > 0) { + console.error( + `Constructed eval/query dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, + ); + failed = true; + } + } +} + if (failed) { console.error( - "retrieval generalization lint failed: remove repo-specific literals from retrieval integration code", + "retrieval generalization lint failed: remove eval/query dependencies from protected product paths", ); process.exit(1); } console.log( - `lint-retrieval-generalization: ok (${scanDirs.length} retrieval dir(s), ${scanFiles.size} retrieval file(s), ${structuralFiles.size} production file(s), ${bannedPatterns.length} patterns)`, + `lint-retrieval-generalization: ok (${scanDirs.length} retrieval dir(s), ${scanFiles.size} retrieval file(s), ${structuralFiles.size} production file(s), ${protectedNonRustScanFiles.size} protected non-Rust file(s), ${bannedPatterns.length} patterns)`, ); From 82613bad6e16cbe50b3d0444ae1aac1d8917ce79 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 10:39:12 -0400 Subject: [PATCH 02/11] close portable corpus guard gaps --- CHANGELOG.md | 4 +- .../tests/retrieval_generalization_guard.rs | 48 +++++++++++-------- docs/testing/performance-review-playbook.md | 13 +++-- scripts/lint-retrieval-generalization.mjs | 35 ++++++++++---- 4 files changed, 63 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae9f00d..24b34e09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,9 @@ non-Rust product and release-control boundary. Direct and adjacent/split evaluation-corpus dependencies now fail in plugin launch/setup surfaces, worktree/install tooling, runtime configuration, release scripts/workflows, - and the release evaluator; missing protected paths fail closed. + and the release evaluator across native separators, case variants, shell + continuations, and PowerShell continuations; missing protected paths fail + closed. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 7cdd3317..7609df16 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -131,7 +131,7 @@ fn run_lint_with_prompt_script_fixture(contents: &str) -> Output { .expect("run lint with prompt script fixture") } -fn run_lint_with_non_rust_fixture(contents: &str) -> Output { +fn run_lint_with_non_rust_fixture(file_name: &str, contents: &str) -> Output { let repo_root = workspace_root(); let script = lint_script(&repo_root); let fixture_root = TempDir::new().expect("create fixture root"); @@ -140,8 +140,7 @@ fn run_lint_with_non_rust_fixture(contents: &str) -> Output { "pub fn repository_neutral_fixture() {}\n", ) .expect("write neutral Rust fixture"); - std::fs::write(fixture_root.path().join("leaked.mjs"), contents) - .expect("write non-Rust fixture"); + std::fs::write(fixture_root.path().join(file_name), contents).expect("write non-Rust fixture"); let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) @@ -419,26 +418,33 @@ pub const LEAKED_DEPENDENCY: &str = include_str!(concat!("../../benchmarks/", "t #[test] fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { - let output = run_lint_with_non_rust_fixture( - r#" -export const directCorpus = import( - "./fetch-holdout-repos.mjs" -); -export const splitCorpus = ["scripts/", "cross-repo-sourcetrail-queries.mjs"].join(""); -"#, - ); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - !output.status.success(), - "protected non-Rust corpus dependencies must fail lint; stderr={stderr}" - ); - for expected in [ - "fetch-holdout-repos.mjs", - "scriptscrossreposourcetrailqueriesmjs", - ] { + let cases = [ + ( + "leaked.ps1", + "$corpus = \"scripts\\cross-repo-\" + `\n \"sourcetrail-queries.mjs\"\n", + "scriptscrossreposourcetrailqueriesmjs", + ), + ( + "leaked.sh", + "corpus=\"benchmarks/tasks/\"\\\n\"eval-probes.json\"\n", + "benchmarkstasksevalprobesjson", + ), + ( + "leaked.mjs", + "export const corpus = import(\"./FETCH-HOLDOUT-REPOS.MJS\");\n", + "fetch-holdout-repos.mjs", + ), + ]; + for (file_name, contents, expected) in cases { + let output = run_lint_with_non_rust_fixture(file_name, contents); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "protected non-Rust corpus dependency in {file_name} must fail lint; stderr={stderr}" + ); assert!( stderr.to_ascii_lowercase().contains(expected), - "lint failure should identify non-Rust corpus dependency {expected}; stderr={stderr}" + "lint failure should identify {expected} in {file_name}; stderr={stderr}" ); } } diff --git a/docs/testing/performance-review-playbook.md b/docs/testing/performance-review-playbook.md index 42ff9789..98527c24 100644 --- a/docs/testing/performance-review-playbook.md +++ b/docs/testing/performance-review-playbook.md @@ -173,16 +173,19 @@ dependencies on every inventoried evaluation/query corpus: The inventory is executable rather than documentation-only. Supported text and configuration files under `scripts/`, `.github/scripts/`, `.github/workflows/`, the shipped plugin, retrieval Compose configuration, and -native backend metadata enter the protected scan by default. Only the explicit -corpus/proof harness list and test/fixture directories are excluded; missing -required protected or harness paths fail the lint so moves require review. +native backend metadata enter the protected scan by default. The tracked Codex +environment definition is a required protected file. Only the explicit +corpus/proof harness list, the named release-detector unit-test harness, and +test/fixture directories are excluded; there are no blanket test-filename +exceptions. Missing required protected or harness paths fail the lint so moves +require review. | Classification | Protected or allowed surface | Static contract | | --- | --- | --- | -| Product launch and setup | Plugin manifests, hooks, MCP launcher, grounding skill guidance/setup scripts, Codex worktree setup, and the Windows installer | Protected; the configured directories are scanned recursively, while test and fixture directories are excluded. | +| Product launch and setup | Plugin manifests, hooks, MCP launcher, grounding skill guidance/setup scripts, Codex environment/worktree setup, and the Windows installer | Protected; the configured directories are scanned recursively, while test and fixture directories are excluded. | | Runtime configuration | Retrieval Compose configuration and native sidecar backend metadata | Protected; new supported text/config files under those directories enter the scan automatically. | | Release control | Release/auto-release workflows, version detection/checking, package assembly, the release-evidence evaluator, and shared provenance validation | Protected; required files must exist, and protected non-Rust modules cannot import an explicitly classified corpus/proof harness module. | -| Explicit corpus harnesses | Task manifests, packet/repo benchmark drivers and scorers, holdout provisioning, release-candidate measurement, the retrieval-sidecar contract workflow, and test fixtures | Allowed to load evaluation corpora; these paths are evidence producers, not product or release-decision logic. Other scripts and workflows remain protected by default. | +| Explicit corpus and test harnesses | Task manifests, packet/repo benchmark drivers and scorers, holdout provisioning, release-candidate measurement, the retrieval-sidecar contract workflow, the release-detector unit test, and test fixtures | Allowed to load evaluation corpora or exercise protected code; these paths are evidence producers/tests, not product or release-decision logic. Other scripts and workflows remain protected by default. | | Environment and generated inputs | Environment values, downloaded manifests/models, generated evidence, and workflow artifacts | Static code paths are protected where listed above. Runtime values and generated bytes require the existing schema, hash, identity, and provenance checks. | | External processes | Git, Docker/Qdrant, embedding servers, and agent executables | Static command construction is protected where it is part of a listed surface. The external executable's internal behavior is outside this repository scan. | diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index c1706152..ee109b07 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -39,6 +39,7 @@ const protectedNonRustDirs = [ ]; const requiredProtectedNonRustFiles = [ + path.join(repoRoot, ".codex", "environments", "environment.toml"), path.join(repoRoot, "scripts", "codestory-evidence-provenance.mjs"), path.join(repoRoot, "scripts", "codestory-release-evidence-gate.mjs"), path.join(repoRoot, "scripts", "codex-worktree-setup.mjs"), @@ -68,13 +69,14 @@ const corpusHarnessNonRustFiles = new Set([ path.join(repoRoot, "scripts", "score-drill-ledger.mjs"), path.join(repoRoot, "scripts", "setup-retrieval-env.mjs"), path.join(repoRoot, "scripts", "setup-retrieval-env.ps1"), + path.join(repoRoot, ".github", "scripts", "test-detect-codestory-release.py"), path.join(repoRoot, ".github", "workflows", "release-candidate-evidence.yml"), path.join(repoRoot, ".github", "workflows", "retrieval-sidecar-smoke.yml"), ].map((filePath) => path.resolve(filePath))); const corpusHarnessModuleNames = new Set( [...corpusHarnessNonRustFiles] .filter((filePath) => [".cjs", ".js", ".mjs"].includes(path.extname(filePath))) - .map((filePath) => path.basename(filePath)), + .map((filePath) => path.basename(filePath).toLowerCase()), ); const protectedNonRustExtensions = new Set([ @@ -654,12 +656,9 @@ function walkProtectedNonRustFiles(root) { return false; } const segments = path.relative(repoRoot, filePath).split(path.sep); - const baseName = path.basename(filePath); return ( !segments.includes("tests") && !segments.includes("fixtures") - && !baseName.startsWith("test-") - && !baseName.includes(".test.") ); }); } @@ -1168,8 +1167,8 @@ function scanCorpusHarnessImports(prepared) { let match; while ((match = importPattern.exec(prepared.production)) != null) { const moduleName = path.posix.basename( - match[1].split(/[?#]/, 1)[0].replaceAll("\\", "/"), - ); + normalizeNativeSeparators(match[1]).split(/[?#]/, 1)[0], + ).toLowerCase(); if (corpusHarnessModuleNames.has(moduleName)) { const line = prepared.production.slice(0, match.index).split(/\r?\n/).length; hits.push( @@ -1184,11 +1183,12 @@ function scanProductionFile(prepared, patterns, combinedRe) { const lines = prepared.lines; const hitsByPattern = new Map(); for (let index = 0; index < lines.length; index += 1) { - if (!combinedRe.test(lines[index])) { + const normalizedLine = normalizeNativeSeparators(lines[index]); + if (!combinedRe.test(normalizedLine)) { continue; } for (const { pattern, re } of patterns) { - if (re.test(lines[index]) && !lineAllowedForPattern(pattern, lines[index])) { + if (re.test(normalizedLine) && !lineAllowedForPattern(pattern, lines[index])) { if (!hitsByPattern.has(pattern)) { hitsByPattern.set(pattern, []); } @@ -1204,7 +1204,10 @@ function scanProductionStringLiterals(prepared, pattern, re) { const hits = []; for (let index = 0; index < lines.length; index += 1) { for (const literal of staticStringLiteralsOnLine(lines[index])) { - if (re.test(literal) && !lineAllowedForPattern(pattern, lines[index])) { + if ( + re.test(normalizeNativeSeparators(literal)) + && !lineAllowedForPattern(pattern, lines[index]) + ) { hits.push(`${prepared.filePath}:${index + 1}:${lines[index]}`); break; } @@ -1220,6 +1223,10 @@ function compactProductionSource(text) { .toLowerCase(); } +function normalizeNativeSeparators(text) { + return text.replaceAll("\\", "/"); +} + function staticStringLiteralContent(literal) { const raw = literal.match(/^b?r(#+)?"([\s\S]*)"(#*)$/); if (raw && (raw[1] ?? "") === raw[3]) { @@ -1302,7 +1309,10 @@ function lineNumberAtOffset(lineStarts, offset) { } function literalJoinGapAllowsCompactScan(gap) { - const withoutJoinCalls = gap + const withoutContinuations = gap + .replace(/\\\r?\n/g, "") + .replace(/`\r?\n/g, ""); + const withoutJoinCalls = withoutContinuations .replace(/\.(?:concat|join)\s*\(/g, "") .replace(/\bpath\.(?:join|resolve)\s*\(/g, ""); return /^[\s,+()[\].]*$/.test(withoutJoinCalls); @@ -1459,6 +1469,11 @@ for (const root of nonRustScanRoots) { protectedNonRustScanFiles.add(filePath); } } +if (usesDefaultNonRustScanRoots) { + for (const filePath of requiredProtectedNonRustFiles) { + protectedNonRustScanFiles.add(filePath); + } +} if (protectedNonRustScanFiles.size === 0) { console.error("lint-retrieval-generalization: no protected non-Rust files found"); process.exit(2); From a82868b2338a736ead9b6bad8aa2f1496cbb6ac1 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 10:48:46 -0400 Subject: [PATCH 03/11] close split harness dependency gaps --- .../tests/retrieval_generalization_guard.rs | 10 +++++ scripts/lint-retrieval-generalization.mjs | 45 +++++++++++++------ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 7609df16..a3f59ba2 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -434,6 +434,16 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "export const corpus = import(\"./FETCH-HOLDOUT-REPOS.MJS\");\n", "fetch-holdout-repos.mjs", ), + ( + "split-import.mjs", + "export const corpus = import(\"./fetch-\" + \"holdout-repos.mjs\");\n", + "fetchholdoutreposmjs", + ), + ( + "split-command.sh", + "script=\"scripts/fetch-\"\\\n\"holdout-repos.mjs\"\nnode \"$script\"\n", + "scriptsfetchholdoutreposmjs", + ), ]; for (file_name, contents, expected) in cases { let output = run_lint_with_non_rust_fixture(file_name, contents); diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index ee109b07..fb842de1 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -73,11 +73,6 @@ const corpusHarnessNonRustFiles = new Set([ path.join(repoRoot, ".github", "workflows", "release-candidate-evidence.yml"), path.join(repoRoot, ".github", "workflows", "retrieval-sidecar-smoke.yml"), ].map((filePath) => path.resolve(filePath))); -const corpusHarnessModuleNames = new Set( - [...corpusHarnessNonRustFiles] - .filter((filePath) => [".cjs", ".js", ".mjs"].includes(path.extname(filePath))) - .map((filePath) => path.basename(filePath).toLowerCase()), -); const protectedNonRustExtensions = new Set([ ".cjs", ".js", ".json", ".md", ".mjs", ".ps1", ".py", ".sh", ".toml", ".ts", @@ -198,9 +193,16 @@ if (scanDirs.length === 0 && productionOnlyFiles.length === 0) { } const evalCorpusBoundaryPatternList = evalCorpusBoundaryPatterns(); -const evalCorpusCompactPatternList = evalCorpusCompactPatterns( +const evalCorpusCompactPatternList = compactBoundaryPatterns( evalCorpusBoundaryPatternList, ); +const corpusHarnessDependencyPatternList = corpusHarnessDependencyPatterns(); +const corpusHarnessDependencyRegexes = corpusHarnessDependencyPatternList.map( + (pattern) => new RegExp(`(?:^|/)${pattern}$`, "i"), +); +const corpusHarnessCompactPatternList = compactBoundaryPatterns( + corpusHarnessDependencyPatternList, +); const bannedPatterns = [ "payload_config", @@ -351,7 +353,17 @@ function evalCorpusBoundaryPatterns() { ].map(escapeRegExp); } -function evalCorpusCompactPatterns(boundaryPatterns) { +function corpusHarnessDependencyPatterns() { + const paths = new Set( + [...corpusHarnessNonRustFiles].flatMap((filePath) => [ + path.relative(repoRoot, filePath).replaceAll(path.sep, "/"), + path.basename(filePath), + ]), + ); + return [...paths].map(escapeRegExp); +} + +function compactBoundaryPatterns(boundaryPatterns) { return boundaryPatterns .map((pattern) => compactProductionSource(pattern.replaceAll("\\", ""))) .filter((pattern) => pattern.length >= 12); @@ -1166,10 +1178,8 @@ function scanCorpusHarnessImports(prepared) { const importPattern = /(?:\bfrom\s+|\bimport\s*\(?\s*|\brequire\s*\(\s*)["'`]([^"'`]+)["'`]/g; let match; while ((match = importPattern.exec(prepared.production)) != null) { - const moduleName = path.posix.basename( - normalizeNativeSeparators(match[1]).split(/[?#]/, 1)[0], - ).toLowerCase(); - if (corpusHarnessModuleNames.has(moduleName)) { + const modulePath = normalizeNativeSeparators(match[1]).split(/[?#]/, 1)[0]; + if (corpusHarnessDependencyRegexes.some((pattern) => pattern.test(modulePath))) { const line = prepared.production.slice(0, match.index).split(/\r?\n/).length; hits.push( `${prepared.filePath}:${line}:${match[0].replace(/\s+/g, " ")}`, @@ -1238,7 +1248,7 @@ function staticStringLiteralContent(literal) { return literal; } -function scanProductionCompactPatterns(prepared, marker) { +function scanProductionCompactPatterns(prepared, marker, minimumLiteralCount = 1) { const production = prepared.production; const markerLower = marker.toLowerCase(); const hits = []; @@ -1258,7 +1268,7 @@ function scanProductionCompactPatterns(prepared, marker) { break; } compact += compactProductionSource(literals[end].literal); - if (compact === markerLower) { + if (end - start + 1 >= minimumLiteralCount && compact === markerLower) { hits.push( compactPatternHit(prepared.filePath, literals[start].line, literals[end].line, marker), ); @@ -1511,6 +1521,15 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { failed = true; } } + for (const pattern of corpusHarnessCompactPatternList) { + const hits = scanProductionCompactPatterns(prepared, pattern, 2); + if (hits.length > 0) { + console.error( + `Constructed evaluation/proof harness dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, + ); + failed = true; + } + } } if (failed) { From 0cf43caefdd3cf084281a5cd900cb5f24f43873f Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 11:01:15 -0400 Subject: [PATCH 04/11] close corpus guard review gaps --- .github/workflows/retrieval-sidecar-smoke.yml | 74 +++++-------------- CHANGELOG.md | 7 +- .../tests/retrieval_generalization_guard.rs | 38 +++++++++- scripts/lint-retrieval-generalization.mjs | 55 +++++++++++++- 4 files changed, 111 insertions(+), 63 deletions(-) diff --git a/.github/workflows/retrieval-sidecar-smoke.yml b/.github/workflows/retrieval-sidecar-smoke.yml index 7586f38f..67552110 100644 --- a/.github/workflows/retrieval-sidecar-smoke.yml +++ b/.github/workflows/retrieval-sidecar-smoke.yml @@ -6,68 +6,30 @@ name: retrieval-sidecar-smoke on: pull_request: paths: - - crates/codestory-retrieval/** - - crates/codestory-contracts/** - - crates/codestory-store/Cargo.toml - - crates/codestory-store/src/** - - crates/codestory-cli/src/retrieval.rs - - crates/codestory-cli/src/main.rs - - crates/codestory-cli/src/args.rs - - crates/codestory-cli/src/runtime.rs - - crates/codestory-cli/src/stdio_*.rs - - crates/codestory-cli/tests/fixtures/packet_search_eval/** - - crates/codestory-cli/tests/packet_search_eval.rs - - crates/codestory-cli/tests/retrieval_bootstrap_contracts.rs - - crates/codestory-cli/tests/search_json_output.rs - - crates/codestory-cli/tests/stdio_protocol_contracts.rs - - crates/codestory-runtime/src/** - - crates/codestory-indexer/Cargo.toml - - crates/codestory-indexer/src/lib.rs - - scripts/lint-retrieval-generalization.mjs - - scripts/codestory-evidence-provenance.mjs - - scripts/codestory-release-evidence-gate.mjs - - scripts/tests/codestory-release-evidence-gate.test.mjs - - benchmarks/release-evidence/** - - scripts/**retrieval** - - docs/ops/retrieval-sidecars.md - - docs/architecture/retrieval-*.md - - docs/testing/retrieval-architecture.md - - docker/retrieval-compose.yml - - .github/workflows/retrieval-sidecar-smoke.yml + - .codex/environments/** + - .github/scripts/** + - .github/workflows/** + - benchmarks/** + - crates/** + - docker/** + - docs/** + - plugins/codestory/** + - scripts/** # Base-branch runs seed caches that sibling PRs are allowed to restore. push: branches: - main - dev/codestory-next paths: - - crates/codestory-retrieval/** - - crates/codestory-contracts/** - - crates/codestory-store/Cargo.toml - - crates/codestory-store/src/** - - crates/codestory-cli/src/retrieval.rs - - crates/codestory-cli/src/main.rs - - crates/codestory-cli/src/args.rs - - crates/codestory-cli/src/runtime.rs - - crates/codestory-cli/src/stdio_*.rs - - crates/codestory-cli/tests/fixtures/packet_search_eval/** - - crates/codestory-cli/tests/packet_search_eval.rs - - crates/codestory-cli/tests/retrieval_bootstrap_contracts.rs - - crates/codestory-cli/tests/search_json_output.rs - - crates/codestory-cli/tests/stdio_protocol_contracts.rs - - crates/codestory-runtime/src/** - - crates/codestory-indexer/Cargo.toml - - crates/codestory-indexer/src/lib.rs - - scripts/lint-retrieval-generalization.mjs - - scripts/codestory-evidence-provenance.mjs - - scripts/codestory-release-evidence-gate.mjs - - scripts/tests/codestory-release-evidence-gate.test.mjs - - benchmarks/release-evidence/** - - scripts/**retrieval** - - docs/ops/retrieval-sidecars.md - - docs/architecture/retrieval-*.md - - docs/testing/retrieval-architecture.md - - docker/retrieval-compose.yml - - .github/workflows/retrieval-sidecar-smoke.yml + - .codex/environments/** + - .github/scripts/** + - .github/workflows/** + - benchmarks/** + - crates/** + - docker/** + - docs/** + - plugins/codestory/** + - scripts/** workflow_dispatch: concurrency: diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b34e09..756868ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,9 +20,10 @@ non-Rust product and release-control boundary. Direct and adjacent/split evaluation-corpus dependencies now fail in plugin launch/setup surfaces, worktree/install tooling, runtime configuration, release scripts/workflows, - and the release evaluator across native separators, case variants, shell - continuations, and PowerShell continuations; missing protected paths fail - closed. + and the release evaluator across native separators, case variants, + comment-separated JavaScript imports, unquoted shell continuations, and + PowerShell continuations. The CI trigger covers the full protected inventory, + and missing protected paths fail closed. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index a3f59ba2..806d6ea9 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -423,31 +423,67 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "leaked.ps1", "$corpus = \"scripts\\cross-repo-\" + `\n \"sourcetrail-queries.mjs\"\n", "scriptscrossreposourcetrailqueriesmjs", + true, ), ( "leaked.sh", "corpus=\"benchmarks/tasks/\"\\\n\"eval-probes.json\"\n", "benchmarkstasksevalprobesjson", + true, ), ( "leaked.mjs", "export const corpus = import(\"./FETCH-HOLDOUT-REPOS.MJS\");\n", "fetch-holdout-repos.mjs", + true, ), ( "split-import.mjs", "export const corpus = import(\"./fetch-\" + \"holdout-repos.mjs\");\n", "fetchholdoutreposmjs", + true, + ), + ( + "commented-import.mjs", + "export const corpus = import(/* deliberate */ \"./fetch-holdout-repos.mjs\");\n", + "fetch-holdout-repos.mjs", + true, + ), + ( + "commented-require.cjs", + "const corpus = require(// deliberate\n \"./fetch-holdout-repos.mjs\");\n", + "fetch-holdout-repos.mjs", + true, ), ( "split-command.sh", "script=\"scripts/fetch-\"\\\n\"holdout-repos.mjs\"\nnode \"$script\"\n", "scriptsfetchholdoutreposmjs", + true, + ), + ( + "unquoted-command.sh", + "script=scripts/fetch-\\\nholdout-repos.mjs\nnode \"$script\"\n", + "scriptsfetchholdoutreposmjs", + true, + ), + ( + "prose.md", + "Do not import(\"./fetch-holdout-repos.mjs\") from production code.\n", + "", + false, ), ]; - for (file_name, contents, expected) in cases { + for (file_name, contents, expected, should_reject) in cases { let output = run_lint_with_non_rust_fixture(file_name, contents); let stderr = String::from_utf8_lossy(&output.stderr); + if !should_reject { + assert!( + output.status.success(), + "non-executable prose in {file_name} must pass lint; stderr={stderr}" + ); + continue; + } assert!( !output.status.success(), "protected non-Rust corpus dependency in {file_name} must fail lint; stderr={stderr}" diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index fb842de1..3c3770c4 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -74,9 +74,12 @@ const corpusHarnessNonRustFiles = new Set([ path.join(repoRoot, ".github", "workflows", "retrieval-sidecar-smoke.yml"), ].map((filePath) => path.resolve(filePath))); +const executableJavaScriptExtensions = new Set([ + ".cjs", ".js", ".mjs", ".ts", ".tsx", +]); const protectedNonRustExtensions = new Set([ - ".cjs", ".js", ".json", ".md", ".mjs", ".ps1", ".py", ".sh", ".toml", ".ts", - ".tsx", ".yaml", ".yml", + ...executableJavaScriptExtensions, + ".json", ".md", ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", ]); const defaultNonRustScanRoots = protectedNonRustDirs; @@ -1174,8 +1177,15 @@ function prepareNonRustFile(filePath) { } function scanCorpusHarnessImports(prepared) { + if (!executableJavaScriptExtensions.has(path.extname(prepared.filePath).toLowerCase())) { + return []; + } const hits = []; - const importPattern = /(?:\bfrom\s+|\bimport\s*\(?\s*|\brequire\s*\(\s*)["'`]([^"'`]+)["'`]/g; + const trivia = String.raw`(?:\s|\/\*[\s\S]*?\*\/|\/\/[^\r\n]*(?:\r?\n|$))*`; + const importPattern = new RegExp( + `(?:\\bfrom${trivia}|\\bimport${trivia}\\(?${trivia}|\\brequire${trivia}\\(${trivia})["'\`]([^"'\`]+)["'\`]`, + "g", + ); let match; while ((match = importPattern.exec(prepared.production)) != null) { const modulePath = normalizeNativeSeparators(match[1]).split(/[?#]/, 1)[0]; @@ -1346,6 +1356,38 @@ function staticStringLiteralsOnLine(line) { return literals; } +function scanShellContinuationHarnessDependencies(prepared) { + if (path.extname(prepared.filePath).toLowerCase() !== ".sh") { + return []; + } + const hits = []; + for (let start = 0; start < prepared.lines.length; start += 1) { + if (!prepared.lines[start].endsWith("\\")) { + continue; + } + let end = start; + let logicalLine = prepared.lines[end].slice(0, -1); + while (end + 1 < prepared.lines.length) { + end += 1; + const continues = prepared.lines[end].endsWith("\\"); + logicalLine += continues + ? prepared.lines[end].slice(0, -1) + : prepared.lines[end]; + if (!continues) { + break; + } + } + const compactLine = logicalLine.replace(/[^a-zA-Z0-9]+/g, "").toLowerCase(); + for (const marker of corpusHarnessCompactPatternList) { + if (compactLine.includes(marker)) { + hits.push(compactPatternHit(prepared.filePath, start + 1, end + 1, marker)); + } + } + start = end; + } + return hits; +} + function lineAllowedForPattern(pattern, line) { return allowedPatternLines.some( (allowed) => allowed.pattern === pattern && line.includes(allowed.includes), @@ -1498,6 +1540,13 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { ); failed = true; } + const shellContinuationHits = scanShellContinuationHarnessDependencies(prepared); + if (shellContinuationHits.length > 0) { + console.error( + `Protected shell path constructs an evaluation/proof harness dependency ${path.relative(repoRoot, filePath)}:\n${shellContinuationHits.join("\n")}\n`, + ); + failed = true; + } const productionHits = scanProductionFile( prepared, corpusRegexPatterns, From cf8c3801c86ab3a83eb91a476711820bd862883b Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 11:17:37 -0400 Subject: [PATCH 05/11] close remaining corpus guard gaps --- .github/workflows/retrieval-sidecar-smoke.yml | 3 + CHANGELOG.md | 5 +- .../tests/retrieval_generalization_guard.rs | 24 +++++++ scripts/lint-retrieval-generalization.mjs | 64 ++++++++++++++----- 4 files changed, 77 insertions(+), 19 deletions(-) diff --git a/.github/workflows/retrieval-sidecar-smoke.yml b/.github/workflows/retrieval-sidecar-smoke.yml index 67552110..29b3ec59 100644 --- a/.github/workflows/retrieval-sidecar-smoke.yml +++ b/.github/workflows/retrieval-sidecar-smoke.yml @@ -79,6 +79,9 @@ jobs: ${{ runner.os }}-cargo-stable-${{ steps.rust-cache-key.outputs.version }}- ${{ runner.os }}-cargo-stable- + - name: Generalization lint regression contracts + run: cargo test -p codestory-runtime --test retrieval_generalization_guard + - name: Runtime sidecar and packet contract tests run: | cargo test -p codestory-runtime --lib agent::retrieval_primary::tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 756868ee..03fd0e90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,9 @@ evaluation-corpus dependencies now fail in plugin launch/setup surfaces, worktree/install tooling, runtime configuration, release scripts/workflows, and the release evaluator across native separators, case variants, - comment-separated JavaScript imports, unquoted shell continuations, and - PowerShell continuations. The CI trigger covers the full protected inventory, + comment-separated JavaScript imports, unquoted shell continuations, + workflow-embedded shell continuations, and PowerShell continuations. The CI + trigger and focused regression contract cover the full protected inventory, and missing protected paths fail closed. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 806d6ea9..fa50e76d 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -443,6 +443,12 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "fetchholdoutreposmjs", true, ), + ( + "comment-split-import.mjs", + "export const corpus = import(\"./fetch-\" /* deliberate */ + \"holdout-repos.mjs\");\n", + "fetchholdoutreposmjs", + true, + ), ( "commented-import.mjs", "export const corpus = import(/* deliberate */ \"./fetch-holdout-repos.mjs\");\n", @@ -467,6 +473,24 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "scriptsfetchholdoutreposmjs", true, ), + ( + "unquoted-eval.sh", + "corpus=benchmarks/ta\\\nsks/eval-probes.json\n", + "benchmarkstasks", + true, + ), + ( + "workflow-command.yml", + "run: |\n node scripts/fetch-\\\n holdout-repos.mjs\n", + "scriptsfetchholdoutreposmjs", + true, + ), + ( + "comment-only-import.mjs", + "// Do not import(\"./fetch-holdout-repos.mjs\") from release code.\n", + "", + false, + ), ( "prose.md", "Do not import(\"./fetch-holdout-repos.mjs\") from production code.\n", diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 3c3770c4..815b523e 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -77,6 +77,13 @@ const corpusHarnessNonRustFiles = new Set([ const executableJavaScriptExtensions = new Set([ ".cjs", ".js", ".mjs", ".ts", ".tsx", ]); +const continuationMarkersByExtension = new Map([ + [".sh", "\\"], + [".ps1", "`"], + [".yml", "\\`"], + [".yaml", "\\`"], +]); +const javaScriptStringOrCommentPattern = /("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)|(\/\*[\s\S]*?\*\/|\/\/[^\r\n]*)/g; const protectedNonRustExtensions = new Set([ ...executableJavaScriptExtensions, ".json", ".md", ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", @@ -206,6 +213,10 @@ const corpusHarnessDependencyRegexes = corpusHarnessDependencyPatternList.map( const corpusHarnessCompactPatternList = compactBoundaryPatterns( corpusHarnessDependencyPatternList, ); +const continuedDependencyCompactPatternList = new Set([ + ...evalCorpusCompactPatternList, + ...corpusHarnessCompactPatternList, +]); const bannedPatterns = [ "payload_config", @@ -1168,26 +1179,34 @@ function prepareProductionFile(filePath) { function prepareNonRustFile(filePath) { const production = readFileSync(filePath, "utf8"); + const extension = path.extname(filePath).toLowerCase(); + const staticSource = executableJavaScriptExtensions.has(extension) + ? maskJavaScriptComments(production) + : production; return { filePath, production, lines: production.split(/\r?\n/), + staticSource, literals: null, }; } +function maskJavaScriptComments(source) { + return source.replace( + javaScriptStringOrCommentPattern, + (token, stringLiteral) => stringLiteral ?? token.replace(/[^\r\n]/g, " "), + ); +} + function scanCorpusHarnessImports(prepared) { if (!executableJavaScriptExtensions.has(path.extname(prepared.filePath).toLowerCase())) { return []; } const hits = []; - const trivia = String.raw`(?:\s|\/\*[\s\S]*?\*\/|\/\/[^\r\n]*(?:\r?\n|$))*`; - const importPattern = new RegExp( - `(?:\\bfrom${trivia}|\\bimport${trivia}\\(?${trivia}|\\brequire${trivia}\\(${trivia})["'\`]([^"'\`]+)["'\`]`, - "g", - ); + const importPattern = /(?:\bfrom\s*|\bimport\s*\(?\s*|\brequire\s*\(\s*)["'`]([^"'`]+)["'`]/g; let match; - while ((match = importPattern.exec(prepared.production)) != null) { + while ((match = importPattern.exec(prepared.staticSource)) != null) { const modulePath = normalizeNativeSeparators(match[1]).split(/[?#]/, 1)[0]; if (corpusHarnessDependencyRegexes.some((pattern) => pattern.test(modulePath))) { const line = prepared.production.slice(0, match.index).split(/\r?\n/).length; @@ -1259,7 +1278,7 @@ function staticStringLiteralContent(literal) { } function scanProductionCompactPatterns(prepared, marker, minimumLiteralCount = 1) { - const production = prepared.production; + const production = prepared.staticSource ?? prepared.production; const markerLower = marker.toLowerCase(); const hits = []; if (prepared.literals == null) { @@ -1356,20 +1375,25 @@ function staticStringLiteralsOnLine(line) { return literals; } -function scanShellContinuationHarnessDependencies(prepared) { - if (path.extname(prepared.filePath).toLowerCase() !== ".sh") { +function scanContinuationDependencies(prepared) { + const continuationMarkers = continuationMarkersByExtension.get( + path.extname(prepared.filePath).toLowerCase(), + ); + if (continuationMarkers == null) { return []; } const hits = []; for (let start = 0; start < prepared.lines.length; start += 1) { - if (!prepared.lines[start].endsWith("\\")) { + const firstMarker = prepared.lines[start].at(-1); + if (!continuationMarkers.includes(firstMarker)) { continue; } let end = start; let logicalLine = prepared.lines[end].slice(0, -1); while (end + 1 < prepared.lines.length) { end += 1; - const continues = prepared.lines[end].endsWith("\\"); + const marker = prepared.lines[end].at(-1); + const continues = continuationMarkers.includes(marker); logicalLine += continues ? prepared.lines[end].slice(0, -1) : prepared.lines[end]; @@ -1378,9 +1402,15 @@ function scanShellContinuationHarnessDependencies(prepared) { } } const compactLine = logicalLine.replace(/[^a-zA-Z0-9]+/g, "").toLowerCase(); - for (const marker of corpusHarnessCompactPatternList) { - if (compactLine.includes(marker)) { - hits.push(compactPatternHit(prepared.filePath, start + 1, end + 1, marker)); + const compactPhysicalLines = prepared.lines + .slice(start, end + 1) + .map((line) => line.replace(/[^a-zA-Z0-9]+/g, "").toLowerCase()); + for (const pattern of continuedDependencyCompactPatternList) { + if ( + compactLine.includes(pattern) + && !compactPhysicalLines.some((line) => line.includes(pattern)) + ) { + hits.push(compactPatternHit(prepared.filePath, start + 1, end + 1, pattern)); } } start = end; @@ -1540,10 +1570,10 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { ); failed = true; } - const shellContinuationHits = scanShellContinuationHarnessDependencies(prepared); - if (shellContinuationHits.length > 0) { + const continuationHits = scanContinuationDependencies(prepared); + if (continuationHits.length > 0) { console.error( - `Protected shell path constructs an evaluation/proof harness dependency ${path.relative(repoRoot, filePath)}:\n${shellContinuationHits.join("\n")}\n`, + `Protected non-Rust path constructs a continued evaluation/proof dependency ${path.relative(repoRoot, filePath)}:\n${continuationHits.join("\n")}\n`, ); failed = true; } From 37f2b4e4294d37688737cea4191083ac87581f77 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 11:53:35 -0400 Subject: [PATCH 06/11] close non-rust harness guard gaps --- CHANGELOG.md | 10 +- .../tests/retrieval_generalization_guard.rs | 34 ++++- scripts/lint-retrieval-generalization.mjs | 137 ++++++++++++++---- 3 files changed, 145 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03fd0e90..e6bca893 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,10 +21,12 @@ evaluation-corpus dependencies now fail in plugin launch/setup surfaces, worktree/install tooling, runtime configuration, release scripts/workflows, and the release evaluator across native separators, case variants, - comment-separated JavaScript imports, unquoted shell continuations, - workflow-embedded shell continuations, and PowerShell continuations. The CI - trigger and focused regression contract cover the full protected inventory, - and missing protected paths fail closed. + direct harness commands/configuration, comment-separated JavaScript imports, + unquoted shell continuations, workflow-embedded shell continuations, and + PowerShell continuations. Comment-only JavaScript and hash-commented + shell/config text is ignored, while Markdown prose stays outside dependency + scanning. The CI trigger and focused regression contract cover the full + protected inventory, and missing protected paths fail closed. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index fa50e76d..d4f17c84 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -487,13 +487,43 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { ), ( "comment-only-import.mjs", - "// Do not import(\"./fetch-holdout-repos.mjs\") from release code.\n", + "// Do not import(\"./fetch-holdout-repos.mjs\") or benchmarks/tasks/eval-probes.json from release code.\n", + "", + false, + ), + ( + "direct-command.sh", + "node scripts/fetch-holdout-repos.mjs\n", + "scripts/fetch-holdout-repos.mjs", + true, + ), + ( + "direct-workflow-command.yml", + "run: node scripts/fetch-holdout-repos.mjs\n", + "scripts/fetch-holdout-repos.mjs", + true, + ), + ( + "spawn-array.mjs", + "spawn(\"node\", [\"scripts/fetch-holdout-repos.mjs\"]);\n", + "scripts/fetch-holdout-repos.mjs", + true, + ), + ( + "one-literal-config.toml", + "harness = \"# scripts/fetch-holdout-repos.mjs\"\n", + "scripts/fetch-holdout-repos.mjs", + true, + ), + ( + "comment-only-continuation.yml", + "# run: node scripts/fetch-\\\n# holdout-repos.mjs\nrun: echo clean\n", "", false, ), ( "prose.md", - "Do not import(\"./fetch-holdout-repos.mjs\") from production code.\n", + "Do not import(\"./fetch-holdout-repos.mjs\") or compose \"fetch-\" + \"holdout-repos.mjs\" in production prose.\n", "", false, ), diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 815b523e..77f4f2cf 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -73,6 +73,12 @@ const corpusHarnessNonRustFiles = new Set([ path.join(repoRoot, ".github", "workflows", "release-candidate-evidence.yml"), path.join(repoRoot, ".github", "workflows", "retrieval-sidecar-smoke.yml"), ].map((filePath) => path.resolve(filePath))); +const corpusSupportNonRustFiles = new Set([ + path.join(repoRoot, "scripts", "lint-retrieval-generalization.mjs"), + path.join(repoRoot, "scripts", "setup-retrieval-env.mjs"), + path.join(repoRoot, "scripts", "setup-retrieval-env.ps1"), + path.join(repoRoot, ".github", "scripts", "test-detect-codestory-release.py"), +].map((filePath) => path.resolve(filePath))); const executableJavaScriptExtensions = new Set([ ".cjs", ".js", ".mjs", ".ts", ".tsx", @@ -83,6 +89,9 @@ const continuationMarkersByExtension = new Map([ [".yml", "\\`"], [".yaml", "\\`"], ]); +const hashCommentExtensions = new Set([ + ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", +]); const javaScriptStringOrCommentPattern = /("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)|(\/\*[\s\S]*?\*\/|\/\/[^\r\n]*)/g; const protectedNonRustExtensions = new Set([ ...executableJavaScriptExtensions, @@ -208,7 +217,7 @@ const evalCorpusCompactPatternList = compactBoundaryPatterns( ); const corpusHarnessDependencyPatternList = corpusHarnessDependencyPatterns(); const corpusHarnessDependencyRegexes = corpusHarnessDependencyPatternList.map( - (pattern) => new RegExp(`(?:^|/)${pattern}$`, "i"), + (pattern) => new RegExp(`(?:^|[^a-z0-9_.-])${pattern}(?=$|[^a-z0-9_.-])`, "i"), ); const corpusHarnessCompactPatternList = compactBoundaryPatterns( corpusHarnessDependencyPatternList, @@ -307,6 +316,7 @@ const bannedPatterns = [ "nvm", "install\\.sh\\s+nvm", "bash_completion\\s+__nvm", + "--with-holdout-clone", ...evalCorpusBoundaryPatternList, ...benchmarkManifestDerivedPatterns(), ...benchmarkEvalProbeDerivedPatterns(), @@ -369,10 +379,12 @@ function evalCorpusBoundaryPatterns() { function corpusHarnessDependencyPatterns() { const paths = new Set( - [...corpusHarnessNonRustFiles].flatMap((filePath) => [ - path.relative(repoRoot, filePath).replaceAll(path.sep, "/"), - path.basename(filePath), - ]), + [...corpusHarnessNonRustFiles] + .filter((filePath) => !corpusSupportNonRustFiles.has(filePath)) + .flatMap((filePath) => [ + path.relative(repoRoot, filePath).replaceAll(path.sep, "/"), + path.basename(filePath), + ]), ); return [...paths].map(escapeRegExp); } @@ -1180,18 +1192,27 @@ function prepareProductionFile(filePath) { function prepareNonRustFile(filePath) { const production = readFileSync(filePath, "utf8"); const extension = path.extname(filePath).toLowerCase(); - const staticSource = executableJavaScriptExtensions.has(extension) - ? maskJavaScriptComments(production) - : production; + const staticSource = maskNonRustComments(production, extension); return { filePath, production, - lines: production.split(/\r?\n/), + lines: staticSource.split(/\r?\n/), + sourceLines: production.split(/\r?\n/), staticSource, literals: null, }; } +function maskNonRustComments(source, extension) { + if (executableJavaScriptExtensions.has(extension)) { + return maskJavaScriptComments(source); + } + if (hashCommentExtensions.has(extension)) { + return maskHashComments(source, extension); + } + return source; +} + function maskJavaScriptComments(source) { return source.replace( javaScriptStringOrCommentPattern, @@ -1199,27 +1220,81 @@ function maskJavaScriptComments(source) { ); } -function scanCorpusHarnessImports(prepared) { - if (!executableJavaScriptExtensions.has(path.extname(prepared.filePath).toLowerCase())) { +function maskHashComments(source, extension) { + const masked = source.split(""); + const escape = extension === ".ps1" ? "`" : "\\"; + const outsideEscape = extension === ".ps1" || extension === ".sh"; + const doubledQuoteEscape = extension === ".ps1" || extension === ".yaml" + || extension === ".yml"; + let quote = null; + + for (let index = 0; index < source.length; index += 1) { + if (quote != null) { + if (source.startsWith(quote, index)) { + if ( + quote.length === 1 + && doubledQuoteEscape + && source[index + 1] === quote + ) { + index += 1; + } else { + index += quote.length - 1; + quote = null; + } + } else if (source[index] === escape && index + 1 < source.length) { + index += 1; + } + continue; + } + + if (outsideEscape && source[index] === escape && index + 1 < source.length) { + index += 1; + continue; + } + if (source[index] === "\"" || source[index] === "'") { + const delimiter = source[index]; + quote = (extension === ".py" || extension === ".toml") + && source.startsWith(delimiter.repeat(3), index) + ? delimiter.repeat(3) + : delimiter; + index += quote.length - 1; + continue; + } + if (source[index] !== "#") { + continue; + } + while (index < source.length && source[index] !== "\r" && source[index] !== "\n") { + masked[index] = " "; + index += 1; + } + index -= 1; + } + return masked.join(""); +} + +function scanCorpusHarnessDependencies(prepared) { + if (!shouldScanCorpusHarnessDependencies(prepared.filePath)) { return []; } const hits = []; - const importPattern = /(?:\bfrom\s*|\bimport\s*\(?\s*|\brequire\s*\(\s*)["'`]([^"'`]+)["'`]/g; - let match; - while ((match = importPattern.exec(prepared.staticSource)) != null) { - const modulePath = normalizeNativeSeparators(match[1]).split(/[?#]/, 1)[0]; - if (corpusHarnessDependencyRegexes.some((pattern) => pattern.test(modulePath))) { - const line = prepared.production.slice(0, match.index).split(/\r?\n/).length; + for (let index = 0; index < prepared.lines.length; index += 1) { + const normalizedLine = normalizeNativeSeparators(prepared.lines[index]); + if (corpusHarnessDependencyRegexes.some((pattern) => pattern.test(normalizedLine))) { hits.push( - `${prepared.filePath}:${line}:${match[0].replace(/\s+/g, " ")}`, + `${prepared.filePath}:${index + 1}:${prepared.sourceLines[index]}`, ); } } return hits; } +function shouldScanCorpusHarnessDependencies(filePath) { + return path.extname(filePath).toLowerCase() !== ".md"; +} + function scanProductionFile(prepared, patterns, combinedRe) { const lines = prepared.lines; + const sourceLines = prepared.sourceLines ?? lines; const hitsByPattern = new Map(); for (let index = 0; index < lines.length; index += 1) { const normalizedLine = normalizeNativeSeparators(lines[index]); @@ -1227,11 +1302,11 @@ function scanProductionFile(prepared, patterns, combinedRe) { continue; } for (const { pattern, re } of patterns) { - if (re.test(normalizedLine) && !lineAllowedForPattern(pattern, lines[index])) { + if (re.test(normalizedLine) && !lineAllowedForPattern(pattern, sourceLines[index])) { if (!hitsByPattern.has(pattern)) { hitsByPattern.set(pattern, []); } - hitsByPattern.get(pattern).push(`${prepared.filePath}:${index + 1}:${lines[index]}`); + hitsByPattern.get(pattern).push(`${prepared.filePath}:${index + 1}:${sourceLines[index]}`); } } } @@ -1563,10 +1638,10 @@ if (protectedNonRustScanFiles.size === 0) { for (const filePath of [...protectedNonRustScanFiles].sort()) { const prepared = prepareNonRustFile(filePath); - const harnessImportHits = scanCorpusHarnessImports(prepared); - if (harnessImportHits.length > 0) { + const harnessDependencyHits = scanCorpusHarnessDependencies(prepared); + if (harnessDependencyHits.length > 0) { console.error( - `Protected non-Rust path imports an evaluation/proof harness ${path.relative(repoRoot, filePath)}:\n${harnessImportHits.join("\n")}\n`, + `Protected non-Rust path depends on an evaluation/proof harness ${path.relative(repoRoot, filePath)}:\n${harnessDependencyHits.join("\n")}\n`, ); failed = true; } @@ -1600,13 +1675,15 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { failed = true; } } - for (const pattern of corpusHarnessCompactPatternList) { - const hits = scanProductionCompactPatterns(prepared, pattern, 2); - if (hits.length > 0) { - console.error( - `Constructed evaluation/proof harness dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, - ); - failed = true; + if (shouldScanCorpusHarnessDependencies(filePath)) { + for (const pattern of corpusHarnessCompactPatternList) { + const hits = scanProductionCompactPatterns(prepared, pattern, 2); + if (hits.length > 0) { + console.error( + `Constructed evaluation/proof harness dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, + ); + failed = true; + } } } } From c6fd4de2c3b322fdec55def7b13813562276b73f Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 12:31:52 -0400 Subject: [PATCH 07/11] tighten retrieval guard parsing --- .github/workflows/retrieval-sidecar-smoke.yml | 18 +- CHANGELOG.md | 7 +- .../tests/retrieval_generalization_guard.rs | 155 ++++------- scripts/lint-retrieval-generalization.mjs | 260 +++++++++++------- 4 files changed, 238 insertions(+), 202 deletions(-) diff --git a/.github/workflows/retrieval-sidecar-smoke.yml b/.github/workflows/retrieval-sidecar-smoke.yml index 29b3ec59..eb64c0e0 100644 --- a/.github/workflows/retrieval-sidecar-smoke.yml +++ b/.github/workflows/retrieval-sidecar-smoke.yml @@ -7,12 +7,19 @@ on: pull_request: paths: - .codex/environments/** + - .github/actions/** - .github/scripts/** - .github/workflows/** - benchmarks/** - crates/** - docker/** - - docs/** + - docs/architecture/language-support.md + - docs/architecture/retrieval-* + - docs/contributors/testing-matrix.md + - docs/ops/retrieval-sidecars.md + - docs/testing/codestory-e2e-stats-log.md + - docs/testing/performance-review-playbook.md + - docs/testing/retrieval-architecture.md - plugins/codestory/** - scripts/** # Base-branch runs seed caches that sibling PRs are allowed to restore. @@ -22,12 +29,19 @@ on: - dev/codestory-next paths: - .codex/environments/** + - .github/actions/** - .github/scripts/** - .github/workflows/** - benchmarks/** - crates/** - docker/** - - docs/** + - docs/architecture/language-support.md + - docs/architecture/retrieval-* + - docs/contributors/testing-matrix.md + - docs/ops/retrieval-sidecars.md + - docs/testing/codestory-e2e-stats-log.md + - docs/testing/performance-review-playbook.md + - docs/testing/retrieval-architecture.md - plugins/codestory/** - scripts/** workflow_dispatch: diff --git a/CHANGELOG.md b/CHANGELOG.md index e6bca893..f7db1607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,9 +24,10 @@ direct harness commands/configuration, comment-separated JavaScript imports, unquoted shell continuations, workflow-embedded shell continuations, and PowerShell continuations. Comment-only JavaScript and hash-commented - shell/config text is ignored, while Markdown prose stays outside dependency - scanning. The CI trigger and focused regression contract cover the full - protected inventory, and missing protected paths fail closed. + shell/config text is ignored, while Markdown prose stays outside evaluation + and dependency scanning. The CI trigger and focused regression contract cover + the full protected inventory, including local composite actions, and missing + protected paths fail closed. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index d4f17c84..c4e119c7 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -131,7 +131,7 @@ fn run_lint_with_prompt_script_fixture(contents: &str) -> Output { .expect("run lint with prompt script fixture") } -fn run_lint_with_non_rust_fixture(file_name: &str, contents: &str) -> Output { +fn run_lint_with_non_rust_fixtures(fixtures: &[(&str, &str)]) -> Output { let repo_root = workspace_root(); let script = lint_script(&repo_root); let fixture_root = TempDir::new().expect("create fixture root"); @@ -140,7 +140,10 @@ fn run_lint_with_non_rust_fixture(file_name: &str, contents: &str) -> Output { "pub fn repository_neutral_fixture() {}\n", ) .expect("write neutral Rust fixture"); - std::fs::write(fixture_root.path().join(file_name), contents).expect("write non-Rust fixture"); + for (file_name, contents) in fixtures { + std::fs::write(fixture_root.path().join(file_name), contents) + .expect("write non-Rust fixture"); + } let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) @@ -418,135 +421,89 @@ pub const LEAKED_DEPENDENCY: &str = include_str!(concat!("../../benchmarks/", "t #[test] fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { - let cases = [ + let rejected = run_lint_with_non_rust_fixtures(&[ ( "leaked.ps1", "$corpus = \"scripts\\cross-repo-\" + `\n \"sourcetrail-queries.mjs\"\n", - "scriptscrossreposourcetrailqueriesmjs", - true, ), ( "leaked.sh", - "corpus=\"benchmarks/tasks/\"\\\n\"eval-probes.json\"\n", - "benchmarkstasksevalprobesjson", - true, - ), - ( - "leaked.mjs", - "export const corpus = import(\"./FETCH-HOLDOUT-REPOS.MJS\");\n", - "fetch-holdout-repos.mjs", - true, - ), - ( - "split-import.mjs", - "export const corpus = import(\"./fetch-\" + \"holdout-repos.mjs\");\n", - "fetchholdoutreposmjs", - true, - ), - ( - "comment-split-import.mjs", - "export const corpus = import(\"./fetch-\" /* deliberate */ + \"holdout-repos.mjs\");\n", - "fetchholdoutreposmjs", - true, + "prefix=./scripts\nscript=${prefix#./}/fetch-holdout-repos.mjs\ncorpus=benchmarks/ta\\\nsks/eval-probes.json\n", ), ( - "commented-import.mjs", - "export const corpus = import(/* deliberate */ \"./fetch-holdout-repos.mjs\");\n", - "fetch-holdout-repos.mjs", - true, + "workflow-command.yml", + "run: |2-\n node scripts/fetch-\\\n holdout-repos.mjs\n", ), ( - "commented-require.cjs", - "const corpus = require(// deliberate\n \"./fetch-holdout-repos.mjs\");\n", - "fetch-holdout-repos.mjs", - true, + "surrounding-command.mjs", + "const command = \"node scripts/fetch-\" + \"holdout-repos.mjs --json\";\nconst config = \"prefix benchmarks/ta\" + \"sks/eval-probes.json suffix\";\n", ), ( - "split-command.sh", - "script=\"scripts/fetch-\"\\\n\"holdout-repos.mjs\"\nnode \"$script\"\n", - "scriptsfetchholdoutreposmjs", - true, + "line-continuation.mjs", + "const script = \"scripts/fetch-holdout-\\\nrepos.mjs\";\n", ), + ]); + let rejected_stderr = String::from_utf8_lossy(&rejected.stderr).to_ascii_lowercase(); + assert!( + !rejected.status.success(), + "executable corpus dependencies must fail lint; stderr={rejected_stderr}" + ); + for (file_name, marker) in [ + ("leaked.ps1", "scriptscrossreposourcetrailqueriesmjs"), + ("leaked.sh", "benchmarkstasksevalprobesjson"), + ("workflow-command.yml", "fetchholdoutreposmjs"), + ("surrounding-command.mjs", "fetchholdoutreposmjs"), + ("line-continuation.mjs", "fetchholdoutreposmjs"), + ] { + assert!( + rejected_stderr.contains(file_name) && rejected_stderr.contains(marker), + "lint failure should identify {file_name} and {marker}; stderr={rejected_stderr}" + ); + } + + let allowed = run_lint_with_non_rust_fixtures(&[ ( - "unquoted-command.sh", - "script=scripts/fetch-\\\nholdout-repos.mjs\nnode \"$script\"\n", - "scriptsfetchholdoutreposmjs", - true, + "prose.md", + "The benchmark harness reads `benchmarks/tasks/eval-probes.json`; production code must not.\n", ), ( - "unquoted-eval.sh", - "corpus=benchmarks/ta\\\nsks/eval-probes.json\n", - "benchmarkstasks", - true, + "quoted-shell.sh", + "value='scripts/fetch-\\\nholdout-repos.mjs'\n", ), ( - "workflow-command.yml", - "run: |\n node scripts/fetch-\\\n holdout-repos.mjs\n", - "scriptsfetchholdoutreposmjs", - true, + "unrelated-list.yml", + "- scripts/fetch-\\\n- holdout-repos.mjs\n", ), ( - "comment-only-import.mjs", - "// Do not import(\"./fetch-holdout-repos.mjs\") or benchmarks/tasks/eval-probes.json from release code.\n", - "", - false, + "template-comment.mjs", + "const value = `${({ clean: true }).clean /* scripts/fetch-holdout-repos.mjs */}`;\n", ), ( - "direct-command.sh", - "node scripts/fetch-holdout-repos.mjs\n", - "scripts/fetch-holdout-repos.mjs", - true, + "quoted-shell-comment.sh", + "value='clean\\' # scripts/fetch-holdout-repos.mjs\n", ), ( - "direct-workflow-command.yml", - "run: node scripts/fetch-holdout-repos.mjs\n", - "scripts/fetch-holdout-repos.mjs", - true, + "quoted-powershell-comment.ps1", + "$value = 'clean`' # scripts/fetch-holdout-repos.mjs\n", ), ( - "spawn-array.mjs", - "spawn(\"node\", [\"scripts/fetch-holdout-repos.mjs\"]);\n", - "scripts/fetch-holdout-repos.mjs", - true, + "quoted-yaml-comment.yml", + "value: 'clean\\' # scripts/fetch-holdout-repos.mjs\n", ), ( - "one-literal-config.toml", - "harness = \"# scripts/fetch-holdout-repos.mjs\"\n", - "scripts/fetch-holdout-repos.mjs", - true, + "folded-workflow.yml", + "run: >-\n node scripts/fetch-\\\n holdout-repos.mjs\n", ), ( - "comment-only-continuation.yml", + "comment-only.yml", "# run: node scripts/fetch-\\\n# holdout-repos.mjs\nrun: echo clean\n", - "", - false, - ), - ( - "prose.md", - "Do not import(\"./fetch-holdout-repos.mjs\") or compose \"fetch-\" + \"holdout-repos.mjs\" in production prose.\n", - "", - false, ), - ]; - for (file_name, contents, expected, should_reject) in cases { - let output = run_lint_with_non_rust_fixture(file_name, contents); - let stderr = String::from_utf8_lossy(&output.stderr); - if !should_reject { - assert!( - output.status.success(), - "non-executable prose in {file_name} must pass lint; stderr={stderr}" - ); - continue; - } - assert!( - !output.status.success(), - "protected non-Rust corpus dependency in {file_name} must fail lint; stderr={stderr}" - ); - assert!( - stderr.to_ascii_lowercase().contains(expected), - "lint failure should identify {expected} in {file_name}; stderr={stderr}" - ); - } + ]); + let allowed_stderr = String::from_utf8_lossy(&allowed.stderr); + assert!( + allowed.status.success(), + "prose, comments, and unrelated continuations must pass lint; stderr={allowed_stderr}" + ); } #[test] diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 77f4f2cf..0ccea5a5 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -31,8 +31,7 @@ const explicitNonRustScanRoots = ( const protectedNonRustDirs = [ path.join(repoRoot, "scripts"), - path.join(repoRoot, ".github", "scripts"), - path.join(repoRoot, ".github", "workflows"), + path.join(repoRoot, ".github"), path.join(repoRoot, "plugins", "codestory"), path.join(repoRoot, "docker"), path.join(repoRoot, "crates", "codestory-retrieval", "assets"), @@ -83,19 +82,14 @@ const corpusSupportNonRustFiles = new Set([ const executableJavaScriptExtensions = new Set([ ".cjs", ".js", ".mjs", ".ts", ".tsx", ]); -const continuationMarkersByExtension = new Map([ - [".sh", "\\"], - [".ps1", "`"], - [".yml", "\\`"], - [".yaml", "\\`"], -]); const hashCommentExtensions = new Set([ ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", ]); -const javaScriptStringOrCommentPattern = /("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)|(\/\*[\s\S]*?\*\/|\/\/[^\r\n]*)/g; +const javaScriptStringOrCommentPattern = /("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*'|`(?:\\[\s\S]|[^`\\])*`)|(\/\*[\s\S]*?\*\/|\/\/[^\r\n]*)/g; +const javaScriptTemplateTokenPattern = /("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*')|(\$\{|[{}])|(\/\*[\s\S]*?\*\/|\/\/[^\r\n]*)/g; const protectedNonRustExtensions = new Set([ ...executableJavaScriptExtensions, - ".json", ".md", ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", + ".json", ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", ]); const defaultNonRustScanRoots = protectedNonRustDirs; @@ -222,10 +216,6 @@ const corpusHarnessDependencyRegexes = corpusHarnessDependencyPatternList.map( const corpusHarnessCompactPatternList = compactBoundaryPatterns( corpusHarnessDependencyPatternList, ); -const continuedDependencyCompactPatternList = new Set([ - ...evalCorpusCompactPatternList, - ...corpusHarnessCompactPatternList, -]); const bannedPatterns = [ "payload_config", @@ -1193,10 +1183,12 @@ function prepareNonRustFile(filePath) { const production = readFileSync(filePath, "utf8"); const extension = path.extname(filePath).toLowerCase(); const staticSource = maskNonRustComments(production, extension); + const lines = staticSource.split(/\r?\n/); return { filePath, production, - lines: staticSource.split(/\r?\n/), + lines, + logicalLines: logicalNonRustLines(lines, extension), sourceLines: production.split(/\r?\n/), staticSource, literals: null, @@ -1216,7 +1208,25 @@ function maskNonRustComments(source, extension) { function maskJavaScriptComments(source) { return source.replace( javaScriptStringOrCommentPattern, - (token, stringLiteral) => stringLiteral ?? token.replace(/[^\r\n]/g, " "), + (token, stringLiteral) => stringLiteral?.startsWith("`") + ? maskTemplateExpressionComments(stringLiteral) + : stringLiteral ?? token.replace(/[^\r\n]/g, " "), + ); +} + +function maskTemplateExpressionComments(template) { + let depth = 0; + return template.replace( + javaScriptTemplateTokenPattern, + (token, stringLiteral, brace, comment) => { + if (stringLiteral != null) return token; + if (brace != null) { + if (brace === "${" || (brace === "{" && depth > 0)) depth += 1; + else if (brace === "}" && depth > 0) depth -= 1; + return token; + } + return depth > 0 ? comment.replace(/[^\r\n]/g, " ") : token; + }, ); } @@ -1241,7 +1251,11 @@ function maskHashComments(source, extension) { index += quote.length - 1; quote = null; } - } else if (source[index] === escape && index + 1 < source.length) { + } else if ( + !quote.startsWith("'") + && source[index] === escape + && index + 1 < source.length + ) { index += 1; } continue; @@ -1263,6 +1277,13 @@ function maskHashComments(source, extension) { if (source[index] !== "#") { continue; } + if ( + extension === ".sh" + && index > 0 + && !/[\s;&|()]/.test(source[index - 1]) + ) { + continue; + } while (index < source.length && source[index] !== "\r" && source[index] !== "\n") { masked[index] = " "; index += 1; @@ -1272,41 +1293,128 @@ function maskHashComments(source, extension) { return masked.join(""); } -function scanCorpusHarnessDependencies(prepared) { - if (!shouldScanCorpusHarnessDependencies(prepared.filePath)) { - return []; +function logicalNonRustLines(lines, extension) { + const yamlScalarKinds = extension === ".yaml" || extension === ".yml" + ? yamlBlockScalarKinds(lines) + : null; + const logicalLines = []; + for (let start = 0; start < lines.length; start += 1) { + let end = start; + let text = lines[end]; + while ( + end + 1 < lines.length + && lineContinuationMarker(text, extension, yamlScalarKinds?.[end]) != null + ) { + text = yamlScalarKinds?.[end] === ">" + ? `${text} ${lines[end + 1].trimStart()}` + : text.slice(0, -1) + lines[end + 1].trimStart(); + end += 1; + } + logicalLines.push({ text, startLine: start + 1, endLine: end + 1 }); + start = end; } + return logicalLines; +} + +function yamlBlockScalarKinds(lines) { + const scalarKinds = Array(lines.length).fill(null); + let blockIndent = null; + let scalarKind = null; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + const indent = line.match(/^\s*/)[0].length; + if (blockIndent != null) { + if (line.trim() === "" || indent > blockIndent) { + scalarKinds[index] = scalarKind; + continue; + } + blockIndent = null; + } + const header = line.trimStart().match( + /^[^#\n]+:\s*([|>])(?:[1-9][+-]?|[+-][1-9]?)?\s*$/, + ); + if (header != null) { + blockIndent = indent; + scalarKind = header[1]; + } + } + return scalarKinds; +} + +function lineContinuationMarker(line, extension, yamlScalarKind) { + let marker = null; + if (executableJavaScriptExtensions.has(extension) || extension === ".sh") { + marker = "\\"; + } else if (extension === ".ps1") { + marker = "`"; + } else if ((extension === ".yaml" || extension === ".yml") && yamlScalarKind != null) { + marker = "\\`".includes(line.at(-1)) ? line.at(-1) : null; + } + if (marker == null || !line.endsWith(marker)) { + return null; + } + const escape = extension === ".ps1" || marker === "`" ? "`" : "\\"; + const quote = openQuoteAtEnd(line.slice(0, -1), escape); + if (quote === "'") { + return null; + } + if (executableJavaScriptExtensions.has(extension) && quote == null) { + return null; + } + return marker; +} + +function openQuoteAtEnd(text, escape) { + let quote = null; + for (let index = 0; index < text.length; index += 1) { + if (quote == null) { + if (text[index] === "\"" || text[index] === "'" || text[index] === "`") { + quote = text[index]; + } + continue; + } + if (quote !== "'" && text[index] === escape) { + index += 1; + } else if (text[index] === quote) { + quote = null; + } + } + return quote; +} + +function scanCorpusHarnessDependencies(prepared) { const hits = []; - for (let index = 0; index < prepared.lines.length; index += 1) { - const normalizedLine = normalizeNativeSeparators(prepared.lines[index]); + for (const line of prepared.logicalLines) { + const normalizedLine = normalizeNativeSeparators(line.text); if (corpusHarnessDependencyRegexes.some((pattern) => pattern.test(normalizedLine))) { hits.push( - `${prepared.filePath}:${index + 1}:${prepared.sourceLines[index]}`, + `${prepared.filePath}:${line.startLine}:${prepared.sourceLines[line.startLine - 1]}`, ); } } return hits; } -function shouldScanCorpusHarnessDependencies(filePath) { - return path.extname(filePath).toLowerCase() !== ".md"; -} - function scanProductionFile(prepared, patterns, combinedRe) { - const lines = prepared.lines; - const sourceLines = prepared.sourceLines ?? lines; + const lines = prepared.logicalLines ?? prepared.lines.map((text, index) => ({ + text, + startLine: index + 1, + endLine: index + 1, + })); + const sourceLines = prepared.sourceLines ?? prepared.lines; const hitsByPattern = new Map(); - for (let index = 0; index < lines.length; index += 1) { - const normalizedLine = normalizeNativeSeparators(lines[index]); + for (const line of lines) { + const normalizedLine = normalizeNativeSeparators(line.text); if (!combinedRe.test(normalizedLine)) { continue; } for (const { pattern, re } of patterns) { - if (re.test(normalizedLine) && !lineAllowedForPattern(pattern, sourceLines[index])) { + const sourceLine = sourceLines[line.startLine - 1]; + if (re.test(normalizedLine) && !lineAllowedForPattern(pattern, sourceLine)) { if (!hitsByPattern.has(pattern)) { hitsByPattern.set(pattern, []); } - hitsByPattern.get(pattern).push(`${prepared.filePath}:${index + 1}:${sourceLines[index]}`); + hitsByPattern.get(pattern).push(`${prepared.filePath}:${line.startLine}:${sourceLine}`); } } } @@ -1352,7 +1460,12 @@ function staticStringLiteralContent(literal) { return literal; } -function scanProductionCompactPatterns(prepared, marker, minimumLiteralCount = 1) { +function scanProductionCompactPatterns( + prepared, + marker, + minimumLiteralCount = 1, + allowSurroundingText = false, +) { const production = prepared.staticSource ?? prepared.production; const markerLower = marker.toLowerCase(); const hits = []; @@ -1372,13 +1485,16 @@ function scanProductionCompactPatterns(prepared, marker, minimumLiteralCount = 1 break; } compact += compactProductionSource(literals[end].literal); - if (end - start + 1 >= minimumLiteralCount && compact === markerLower) { + const containsMarker = allowSurroundingText + ? compact.includes(markerLower) + : compact === markerLower; + if (end - start + 1 >= minimumLiteralCount && containsMarker) { hits.push( compactPatternHit(prepared.filePath, literals[start].line, literals[end].line, marker), ); break; } - if (compact.length >= markerLower.length) { + if (!allowSurroundingText && compact.length >= markerLower.length) { break; } } @@ -1395,7 +1511,7 @@ function staticStringLiteralSpans(text) { } } - const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`)/g; + const stringLiteral = /(?:b?r#*"[^"]*"#*|"(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*'|`(?:\\[\s\S]|[^`\\])*`)/g; let match; while ((match = stringLiteral.exec(text)) != null) { literals.push({ @@ -1450,49 +1566,6 @@ function staticStringLiteralsOnLine(line) { return literals; } -function scanContinuationDependencies(prepared) { - const continuationMarkers = continuationMarkersByExtension.get( - path.extname(prepared.filePath).toLowerCase(), - ); - if (continuationMarkers == null) { - return []; - } - const hits = []; - for (let start = 0; start < prepared.lines.length; start += 1) { - const firstMarker = prepared.lines[start].at(-1); - if (!continuationMarkers.includes(firstMarker)) { - continue; - } - let end = start; - let logicalLine = prepared.lines[end].slice(0, -1); - while (end + 1 < prepared.lines.length) { - end += 1; - const marker = prepared.lines[end].at(-1); - const continues = continuationMarkers.includes(marker); - logicalLine += continues - ? prepared.lines[end].slice(0, -1) - : prepared.lines[end]; - if (!continues) { - break; - } - } - const compactLine = logicalLine.replace(/[^a-zA-Z0-9]+/g, "").toLowerCase(); - const compactPhysicalLines = prepared.lines - .slice(start, end + 1) - .map((line) => line.replace(/[^a-zA-Z0-9]+/g, "").toLowerCase()); - for (const pattern of continuedDependencyCompactPatternList) { - if ( - compactLine.includes(pattern) - && !compactPhysicalLines.some((line) => line.includes(pattern)) - ) { - hits.push(compactPatternHit(prepared.filePath, start + 1, end + 1, pattern)); - } - } - start = end; - } - return hits; -} - function lineAllowedForPattern(pattern, line) { return allowedPatternLines.some( (allowed) => allowed.pattern === pattern && line.includes(allowed.includes), @@ -1612,7 +1685,7 @@ for (const filePath of [...structuralFiles].sort()) { } } for (const pattern of evalCorpusCompactPatternList) { - const hits = scanProductionCompactPatterns(prepared, pattern); + const hits = scanProductionCompactPatterns(prepared, pattern, 1, true); if (hits.length > 0) { console.error(`Constructed production dependency on eval/query corpus /${pattern}/ in ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`); failed = true; @@ -1645,13 +1718,6 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { ); failed = true; } - const continuationHits = scanContinuationDependencies(prepared); - if (continuationHits.length > 0) { - console.error( - `Protected non-Rust path constructs a continued evaluation/proof dependency ${path.relative(repoRoot, filePath)}:\n${continuationHits.join("\n")}\n`, - ); - failed = true; - } const productionHits = scanProductionFile( prepared, corpusRegexPatterns, @@ -1667,7 +1733,7 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { } } for (const pattern of evalCorpusCompactPatternList) { - const hits = scanProductionCompactPatterns(prepared, pattern); + const hits = scanProductionCompactPatterns(prepared, pattern, 1, true); if (hits.length > 0) { console.error( `Constructed eval/query dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, @@ -1675,15 +1741,13 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { failed = true; } } - if (shouldScanCorpusHarnessDependencies(filePath)) { - for (const pattern of corpusHarnessCompactPatternList) { - const hits = scanProductionCompactPatterns(prepared, pattern, 2); - if (hits.length > 0) { - console.error( - `Constructed evaluation/proof harness dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, - ); - failed = true; - } + for (const pattern of corpusHarnessCompactPatternList) { + const hits = scanProductionCompactPatterns(prepared, pattern, 2, true); + if (hits.length > 0) { + console.error( + `Constructed evaluation/proof harness dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, + ); + failed = true; } } } From a3f1b7a818c98245a449b30240e246124be8a3d2 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 12:51:19 -0400 Subject: [PATCH 08/11] close agent guidance guard gaps --- .github/workflows/retrieval-sidecar-smoke.yml | 2 + CHANGELOG.md | 4 +- .../tests/retrieval_generalization_guard.rs | 30 ++++++- scripts/lint-retrieval-generalization.mjs | 81 ++++++++++++++++--- 4 files changed, 102 insertions(+), 15 deletions(-) diff --git a/.github/workflows/retrieval-sidecar-smoke.yml b/.github/workflows/retrieval-sidecar-smoke.yml index eb64c0e0..2aab4c2a 100644 --- a/.github/workflows/retrieval-sidecar-smoke.yml +++ b/.github/workflows/retrieval-sidecar-smoke.yml @@ -7,6 +7,7 @@ on: pull_request: paths: - .codex/environments/** + - .cursor/rules/** - .github/actions/** - .github/scripts/** - .github/workflows/** @@ -29,6 +30,7 @@ on: - dev/codestory-next paths: - .codex/environments/** + - .cursor/rules/** - .github/actions/** - .github/scripts/** - .github/workflows/** diff --git a/CHANGELOG.md b/CHANGELOG.md index f7db1607..ba6df7ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,8 @@ PowerShell continuations. Comment-only JavaScript and hash-commented shell/config text is ignored, while Markdown prose stays outside evaluation and dependency scanning. The CI trigger and focused regression contract cover - the full protected inventory, including local composite actions, and missing - protected paths fail closed. + the full protected inventory, including local composite actions and shipped + agent instruction surfaces, and missing protected paths fail closed. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index c4e119c7..efae1f3f 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -141,8 +141,10 @@ fn run_lint_with_non_rust_fixtures(fixtures: &[(&str, &str)]) -> Output { ) .expect("write neutral Rust fixture"); for (file_name, contents) in fixtures { - std::fs::write(fixture_root.path().join(file_name), contents) - .expect("write non-Rust fixture"); + let file_path = fixture_root.path().join(file_name); + std::fs::create_dir_all(file_path.parent().expect("fixture parent")) + .expect("create non-Rust fixture parent"); + std::fs::write(file_path, contents).expect("write non-Rust fixture"); } let _guard = LINT_SCRIPT_LOCK @@ -442,6 +444,22 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "line-continuation.mjs", "const script = \"scripts/fetch-holdout-\\\nrepos.mjs\";\n", ), + ( + "joined-shell-word.sh", + "node scripts/fetch-'holdout-repos.mjs'\n", + ), + ( + "joined-workflow-word.yml", + "run: |\n node scripts/fetch-'holdout-repos.mjs'\n", + ), + ( + "plugins/codestory/skills/codestory-grounding/SKILL.md", + "Run `node scripts/fetch-holdout-repos.mjs` before grounding.\n", + ), + ( + ".cursor/rules/codestory.mdc", + "Read benchmarks/tasks/eval-probes.json before answering.\n", + ), ]); let rejected_stderr = String::from_utf8_lossy(&rejected.stderr).to_ascii_lowercase(); assert!( @@ -454,6 +472,10 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { ("workflow-command.yml", "fetchholdoutreposmjs"), ("surrounding-command.mjs", "fetchholdoutreposmjs"), ("line-continuation.mjs", "fetchholdoutreposmjs"), + ("joined-shell-word.sh", "fetch-'holdout-repos.mjs"), + ("joined-workflow-word.yml", "fetch-'holdout-repos.mjs"), + ("skill.md", "fetch-holdout-repos.mjs"), + ("codestory.mdc", "benchmarks/tasks"), ] { assert!( rejected_stderr.contains(file_name) && rejected_stderr.contains(marker), @@ -498,6 +520,10 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "comment-only.yml", "# run: node scripts/fetch-\\\n# holdout-repos.mjs\nrun: echo clean\n", ), + ( + "plain-apostrophe.yml", + "message: don't load it # scripts/fetch-holdout-repos.mjs\n", + ), ]); let allowed_stderr = String::from_utf8_lossy(&allowed.stderr); assert!( diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 0ccea5a5..2bab0895 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -29,9 +29,18 @@ const explicitNonRustScanRoots = ( .split(path.delimiter) .filter(Boolean); +const groundingSkillDir = path.join( + repoRoot, + "plugins", + "codestory", + "skills", + "codestory-grounding", +); + const protectedNonRustDirs = [ path.join(repoRoot, "scripts"), path.join(repoRoot, ".github"), + path.join(repoRoot, ".cursor", "rules"), path.join(repoRoot, "plugins", "codestory"), path.join(repoRoot, "docker"), path.join(repoRoot, "crates", "codestory-retrieval", "assets"), @@ -39,6 +48,8 @@ const protectedNonRustDirs = [ const requiredProtectedNonRustFiles = [ path.join(repoRoot, ".codex", "environments", "environment.toml"), + path.join(repoRoot, ".cursor", "rules", "codestory.mdc"), + path.join(groundingSkillDir, "SKILL.md"), path.join(repoRoot, "scripts", "codestory-evidence-provenance.mjs"), path.join(repoRoot, "scripts", "codestory-release-evidence-gate.mjs"), path.join(repoRoot, "scripts", "codex-worktree-setup.mjs"), @@ -78,6 +89,13 @@ const corpusSupportNonRustFiles = new Set([ path.join(repoRoot, "scripts", "setup-retrieval-env.ps1"), path.join(repoRoot, ".github", "scripts", "test-detect-codestory-release.py"), ].map((filePath) => path.resolve(filePath))); +const allowedHarnessDependencyLines = [ + ["drill-suite.md", "scripts/score-drill-ledger.mjs"], + ["retrieval-rollout.md", ".github/workflows/retrieval-sidecar-smoke.yml"], +].map(([fileName, includes]) => ({ + filePath: path.join(groundingSkillDir, "references", fileName), + includes, +})); const executableJavaScriptExtensions = new Set([ ".cjs", ".js", ".mjs", ".ts", ".tsx", @@ -89,8 +107,13 @@ const javaScriptStringOrCommentPattern = /("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[ const javaScriptTemplateTokenPattern = /("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*')|(\$\{|[{}])|(\/\*[\s\S]*?\*\/|\/\/[^\r\n]*)/g; const protectedNonRustExtensions = new Set([ ...executableJavaScriptExtensions, - ".json", ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", + ".json", ".md", ".mdc", ".ps1", ".py", ".sh", ".toml", ".yaml", ".yml", ]); +const agentInstructionExtensions = new Set([".md", ".mdc"]); +const agentInstructionPathFragments = [ + path.join("plugins", "codestory", "skills", "codestory-grounding"), + path.join(".cursor", "rules"), +]; const defaultNonRustScanRoots = protectedNonRustDirs; const usesDefaultNonRustScanRoots = explicitNonRustScanRoots.length === 0; @@ -674,9 +697,16 @@ function walkFiles(root, predicate) { function walkProtectedNonRustFiles(root) { return walkFiles(root, (filePath) => { - if (!protectedNonRustExtensions.has(path.extname(filePath).toLowerCase())) { + const extension = path.extname(filePath).toLowerCase(); + if (!protectedNonRustExtensions.has(extension)) { return false; } + if ( + agentInstructionExtensions.has(extension) + && !agentInstructionPathFragments.some((fragment) => + filePath.includes(`${path.sep}${fragment}${path.sep}`) + ) + ) return false; if (!usesDefaultNonRustScanRoots) { return true; } @@ -1265,6 +1295,13 @@ function maskHashComments(source, extension) { index += 1; continue; } + if ( + source[index] === "'" + && (extension === ".yaml" || extension === ".yml") + && /[a-zA-Z0-9_]/.test(source[index - 1] ?? "") + ) { + continue; + } if (source[index] === "\"" || source[index] === "'") { const delimiter = source[index]; quote = (extension === ".py" || extension === ".toml") @@ -1305,12 +1342,19 @@ function logicalNonRustLines(lines, extension) { end + 1 < lines.length && lineContinuationMarker(text, extension, yamlScalarKinds?.[end]) != null ) { - text = yamlScalarKinds?.[end] === ">" + text = yamlScalarKinds?.[end]?.endsWith(">") ? `${text} ${lines[end + 1].trimStart()}` : text.slice(0, -1) + lines[end + 1].trimStart(); end += 1; } - logicalLines.push({ text, startLine: start + 1, endLine: end + 1 }); + logicalLines.push({ + text, + startLine: start + 1, + endLine: end + 1, + shellLike: extension === ".sh" + || yamlScalarKinds?.[start]?.startsWith("run") + || isYamlRunLine(lines[start], extension), + }); start = end; } return logicalLines; @@ -1331,16 +1375,22 @@ function yamlBlockScalarKinds(lines) { blockIndent = null; } const header = line.trimStart().match( - /^[^#\n]+:\s*([|>])(?:[1-9][+-]?|[+-][1-9]?)?\s*$/, + /^([^#\n]+):\s*([|>])(?:[1-9][+-]?|[+-][1-9]?)?\s*$/, ); if (header != null) { blockIndent = indent; - scalarKind = header[1]; + const shellPrefix = /^(?:-\s*)?run$/.test(header[1].trim()) ? "run" : ""; + scalarKind = `${shellPrefix}${header[2]}`; } } return scalarKinds; } +function isYamlRunLine(line, extension) { + return (extension === ".yaml" || extension === ".yml") + && /^\s*(?:-\s*)?run\s*:/.test(line); +} + function lineContinuationMarker(line, extension, yamlScalarKind) { let marker = null; if (executableJavaScriptExtensions.has(extension) || extension === ".sh") { @@ -1385,8 +1435,16 @@ function openQuoteAtEnd(text, escape) { function scanCorpusHarnessDependencies(prepared) { const hits = []; for (const line of prepared.logicalLines) { - const normalizedLine = normalizeNativeSeparators(line.text); - if (corpusHarnessDependencyRegexes.some((pattern) => pattern.test(normalizedLine))) { + const normalizedLine = normalizeNativeSeparators(line.text, line.shellLike); + const hasProtectedDependency = corpusHarnessDependencyRegexes.some((pattern) => + pattern.test(normalizedLine) + && !allowedHarnessDependencyLines.some(({ filePath, includes }) => + path.resolve(prepared.filePath) === filePath + && line.text.includes(includes) + && pattern.test(includes) + ) + ); + if (hasProtectedDependency) { hits.push( `${prepared.filePath}:${line.startLine}:${prepared.sourceLines[line.startLine - 1]}`, ); @@ -1404,7 +1462,7 @@ function scanProductionFile(prepared, patterns, combinedRe) { const sourceLines = prepared.sourceLines ?? prepared.lines; const hitsByPattern = new Map(); for (const line of lines) { - const normalizedLine = normalizeNativeSeparators(line.text); + const normalizedLine = normalizeNativeSeparators(line.text, line.shellLike); if (!combinedRe.test(normalizedLine)) { continue; } @@ -1445,8 +1503,9 @@ function compactProductionSource(text) { .toLowerCase(); } -function normalizeNativeSeparators(text) { - return text.replaceAll("\\", "/"); +function normalizeNativeSeparators(text, shellLike = false) { + const normalized = text.replaceAll("\\", "/"); + return shellLike ? normalized.replace(/["']/g, "") : normalized; } function staticStringLiteralContent(literal) { From b7f9818f30ba94b02874964d3a49fd074a66f986 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 13:50:41 -0400 Subject: [PATCH 09/11] tighten corpus boundary lint --- .../tests/retrieval_generalization_guard.rs | 33 +++++ scripts/lint-retrieval-generalization.mjs | 138 ++++++++++++++++-- 2 files changed, 155 insertions(+), 16 deletions(-) diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index efae1f3f..7bd8f473 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -452,6 +452,26 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "joined-workflow-word.yml", "run: |\n node scripts/fetch-'holdout-repos.mjs'\n", ), + ( + "quoted-run-key.yml", + "steps:\n - \"run\": |\n node scripts/fetch-'holdout-repos.mjs'\n", + ), + ( + "escaped-shell-word.sh", + "node scripts/fetch\\-holdout-repos.mjs\n", + ), + ( + "quoted-yaml-scalar.yml", + "value: 'clean # scripts/fetch-holdout-repos.mjs'\n", + ), + ( + "quoted-block-scalar.yml", + "run: |\n value='clean # scripts/fetch-holdout-repos.mjs'\n", + ), + ( + "github-script.yml", + "uses: actions/github-script@v8\nwith:\n script: |\n const script = \"scripts/fetch-\" + \"holdout-repos.mjs\";\n", + ), ( "plugins/codestory/skills/codestory-grounding/SKILL.md", "Run `node scripts/fetch-holdout-repos.mjs` before grounding.\n", @@ -474,6 +494,11 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { ("line-continuation.mjs", "fetchholdoutreposmjs"), ("joined-shell-word.sh", "fetch-'holdout-repos.mjs"), ("joined-workflow-word.yml", "fetch-'holdout-repos.mjs"), + ("quoted-run-key.yml", "fetch-'holdout-repos.mjs"), + ("escaped-shell-word.sh", "fetch\\-holdout-repos.mjs"), + ("quoted-yaml-scalar.yml", "fetch-holdout-repos.mjs"), + ("quoted-block-scalar.yml", "fetch-holdout-repos.mjs"), + ("github-script.yml", "fetchholdoutreposmjs"), ("skill.md", "fetch-holdout-repos.mjs"), ("codestory.mdc", "benchmarks/tasks"), ] { @@ -524,6 +549,14 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "plain-apostrophe.yml", "message: don't load it # scripts/fetch-holdout-repos.mjs\n", ), + ( + "punctuated-apostrophe.yml", + "message: rock-'n roll # scripts/fetch-holdout-repos.mjs\n", + ), + ( + "doubled-single-quote.yml", + "value: 'scripts/fetch-''holdout-repos.mjs'\n", + ), ]); let allowed_stderr = String::from_utf8_lossy(&allowed.stderr); assert!( diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 2bab0895..92c205ce 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -1216,6 +1216,7 @@ function prepareNonRustFile(filePath) { const lines = staticSource.split(/\r?\n/); return { filePath, + extension, production, lines, logicalLines: logicalNonRustLines(lines, extension), @@ -1229,6 +1230,9 @@ function maskNonRustComments(source, extension) { if (executableJavaScriptExtensions.has(extension)) { return maskJavaScriptComments(source); } + if (extension === ".yaml" || extension === ".yml") { + return maskYamlComments(source); + } if (hashCommentExtensions.has(extension)) { return maskHashComments(source, extension); } @@ -1264,8 +1268,7 @@ function maskHashComments(source, extension) { const masked = source.split(""); const escape = extension === ".ps1" ? "`" : "\\"; const outsideEscape = extension === ".ps1" || extension === ".sh"; - const doubledQuoteEscape = extension === ".ps1" || extension === ".yaml" - || extension === ".yml"; + const doubledQuoteEscape = extension === ".ps1"; let quote = null; for (let index = 0; index < source.length; index += 1) { @@ -1295,13 +1298,6 @@ function maskHashComments(source, extension) { index += 1; continue; } - if ( - source[index] === "'" - && (extension === ".yaml" || extension === ".yml") - && /[a-zA-Z0-9_]/.test(source[index - 1] ?? "") - ) { - continue; - } if (source[index] === "\"" || source[index] === "'") { const delimiter = source[index]; quote = (extension === ".py" || extension === ".toml") @@ -1330,6 +1326,63 @@ function maskHashComments(source, extension) { return masked.join(""); } +function maskYamlComments(source) { + const masked = source.split(""); + const blockScalarKinds = yamlBlockScalarKinds(source.split(/\r?\n/)); + let quote = null; + let quoteInBlockScalar = false; + let line = 0; + + for (let index = 0; index < source.length; index += 1) { + const current = source[index]; + if (index > 0 && source[index - 1] === "\n") line += 1; + if (quote != null) { + if ( + !quoteInBlockScalar + && quote === "'" + && current === "'" + && source[index + 1] === "'" + ) index += 1; + else if (quote === '"' && current === "\\") index += 1; + else if (current === quote) { + quote = null; + quoteInBlockScalar = false; + } + continue; + } + quoteInBlockScalar = blockScalarKinds[line] != null; + if ( + (current === "'" || current === '"') + && (quoteInBlockScalar || yamlQuoteStartsScalar(source, index)) + ) { + quote = current; + continue; + } + quoteInBlockScalar = false; + if ( + current !== "#" + || (index > 0 && source[index - 1] !== "\n" && !/[\t ]/.test(source[index - 1])) + ) { + continue; + } + while (index < source.length && source[index] !== "\r" && source[index] !== "\n") { + masked[index] = " "; + index += 1; + } + index -= 1; + } + return masked.join(""); +} + +function yamlQuoteStartsScalar(source, index) { + const lineStart = source.lastIndexOf("\n", index - 1) + 1; + const before = source.slice(lineStart, index); + const token = before.trimEnd(); + const delimiter = token.at(-1); + if (delimiter == null || ":[{,?".includes(delimiter)) return true; + return delimiter === "-" && /(?:^|[\t [{,])-$/.test(token); +} + function logicalNonRustLines(lines, extension) { const yamlScalarKinds = extension === ".yaml" || extension === ".yml" ? yamlBlockScalarKinds(lines) @@ -1347,8 +1400,9 @@ function logicalNonRustLines(lines, extension) { : text.slice(0, -1) + lines[end + 1].trimStart(); end += 1; } + const yamlRunCommand = yamlRunCommandText(lines[start], extension); logicalLines.push({ - text, + text: yamlRunCommand ?? text, startLine: start + 1, endLine: end + 1, shellLike: extension === ".sh" @@ -1375,11 +1429,12 @@ function yamlBlockScalarKinds(lines) { blockIndent = null; } const header = line.trimStart().match( - /^([^#\n]+):\s*([|>])(?:[1-9][+-]?|[+-][1-9]?)?\s*$/, + /^([^#\n]+):\s*([|>])(?:[1-9][+-]?|[+-][1-9]?)?(?:\s+#.*)?\s*$/, ); if (header != null) { blockIndent = indent; - const shellPrefix = /^(?:-\s*)?run$/.test(header[1].trim()) ? "run" : ""; + const key = header[1].trim().replace(/^-\s*/, ""); + const shellPrefix = ["run", "'run'", '"run"'].includes(key) ? "run" : ""; scalarKind = `${shellPrefix}${header[2]}`; } } @@ -1388,7 +1443,22 @@ function yamlBlockScalarKinds(lines) { function isYamlRunLine(line, extension) { return (extension === ".yaml" || extension === ".yml") - && /^\s*(?:-\s*)?run\s*:/.test(line); + && yamlRunLineMatch(line) != null; +} + +function yamlRunLineMatch(line) { + return line.match(/^\s*(?:-\s*)?(?:run|'run'|"run")\s*:\s*(.*)$/); +} + +function yamlRunCommandText(line, extension) { + if (extension !== ".yaml" && extension !== ".yml") return null; + const match = yamlRunLineMatch(line); + if (match == null) return null; + const scalar = match[1].trim(); + if (/^[|>](?:[1-9][+-]?|[+-][1-9]?)?$/.test(scalar)) return null; + if (scalar.startsWith("'") && scalar.endsWith("'")) return scalar.slice(1, -1).replaceAll("''", "'"); + if (scalar.startsWith('"') && scalar.endsWith('"')) return scalar.slice(1, -1).replace(/\\(["\\])/g, "$1"); + return scalar; } function lineContinuationMarker(line, extension, yamlScalarKind) { @@ -1504,8 +1574,33 @@ function compactProductionSource(text) { } function normalizeNativeSeparators(text, shellLike = false) { - const normalized = text.replaceAll("\\", "/"); - return shellLike ? normalized.replace(/["']/g, "") : normalized; + const normalized = shellLike ? normalizeShellLikeText(text) : text; + return normalized.replaceAll("\\", "/"); +} + +function normalizeShellLikeText(text) { + let normalized = ""; + let quote = null; + for (let index = 0; index < text.length; index += 1) { + const current = text[index]; + if (quote == null && (current === "'" || current === '"')) { + quote = current; + continue; + } + if (current === quote) { + quote = null; + continue; + } + const escaped = current === "\\" && index + 1 < text.length + && (quote == null || (quote === '"' && '$`"\\'.includes(text[index + 1]))); + if (escaped) { + normalized += text[index + 1]; + index += 1; + } else { + normalized += current; + } + } + return normalized; } function staticStringLiteralContent(literal) { @@ -1539,6 +1634,9 @@ function scanProductionCompactPatterns( end > start && !literalJoinGapAllowsCompactScan( production.slice(literals[end - 1].endOffset, literals[end].startOffset), + prepared.extension, + literals[end - 1].literal, + literals[end].literal, ) ) { break; @@ -1597,7 +1695,15 @@ function lineNumberAtOffset(lineStarts, offset) { return high + 1; } -function literalJoinGapAllowsCompactScan(gap) { +function literalJoinGapAllowsCompactScan(gap, extension, previousLiteral, nextLiteral) { + if ( + (extension === ".yaml" || extension === ".yml") + && gap === "" + && previousLiteral.startsWith("'") + && nextLiteral.startsWith("'") + ) { + return false; + } const withoutContinuations = gap .replace(/\\\r?\n/g, "") .replace(/`\r?\n/g, ""); From f859a238abc983352e78890ec71ba65367b8115d Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 16:08:53 -0400 Subject: [PATCH 10/11] fix corpus guard policy references --- CHANGELOG.md | 5 +- .../tests/retrieval_generalization_guard.rs | 15 +++ scripts/lint-retrieval-generalization.mjs | 104 +++++++++++++++--- 3 files changed, 107 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06c6c036..3bea1a94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,10 @@ shell/config text is ignored, while Markdown prose stays outside evaluation and dependency scanning. The CI trigger and focused regression contract cover the full protected inventory, including local composite actions and shipped - agent instruction surfaces, and missing protected paths fail closed. + agent instruction surfaces, and missing protected paths fail closed. Exact + release-policy references to the retrieval smoke workflow are allowed only + by file and literal, while constructed dependencies must span the literals + that form the matched harness marker. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 7bd8f473..cfe09433 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -472,6 +472,18 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { "github-script.yml", "uses: actions/github-script@v8\nwith:\n script: |\n const script = \"scripts/fetch-\" + \"holdout-repos.mjs\";\n", ), + ( + "direct-harness-import.mjs", + "import \"./scripts/codestory-agent-ab-benchmark.mjs\";\n", + ), + ( + "constructed-harness-import.mjs", + "const harness = \"scripts/codestory-agent-ab-\" + \"benchmark.mjs\";\nawait import(harness);\n", + ), + ( + "unapproved-policy-reference.mjs", + "const workflows = [\".github/workflows/retrieval-sidecar-smoke.yml\", \"unrelated.yml\"];\n", + ), ( "plugins/codestory/skills/codestory-grounding/SKILL.md", "Run `node scripts/fetch-holdout-repos.mjs` before grounding.\n", @@ -499,6 +511,9 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { ("quoted-yaml-scalar.yml", "fetch-holdout-repos.mjs"), ("quoted-block-scalar.yml", "fetch-holdout-repos.mjs"), ("github-script.yml", "fetchholdoutreposmjs"), + ("direct-harness-import.mjs", "codestory-agent-ab-benchmark.mjs"), + ("constructed-harness-import.mjs", "codestoryagentabbenchmarkmjs"), + ("unapproved-policy-reference.mjs", "retrieval-sidecar-smoke.yml"), ("skill.md", "fetch-holdout-repos.mjs"), ("codestory.mdc", "benchmarks/tasks"), ] { diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 92c205ce..c5a2c2cf 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -89,11 +89,25 @@ const corpusSupportNonRustFiles = new Set([ path.join(repoRoot, "scripts", "setup-retrieval-env.ps1"), path.join(repoRoot, ".github", "scripts", "test-detect-codestory-release.py"), ].map((filePath) => path.resolve(filePath))); -const allowedHarnessDependencyLines = [ - ["drill-suite.md", "scripts/score-drill-ledger.mjs"], - ["retrieval-rollout.md", ".github/workflows/retrieval-sidecar-smoke.yml"], -].map(([fileName, includes]) => ({ - filePath: path.join(groundingSkillDir, "references", fileName), +const allowedHarnessReferences = [ + [ + path.join(groundingSkillDir, "references", "drill-suite.md"), + "scripts/score-drill-ledger.mjs", + ], + [ + path.join(groundingSkillDir, "references", "retrieval-rollout.md"), + ".github/workflows/retrieval-sidecar-smoke.yml", + ], + [ + path.join(repoRoot, ".github", "scripts", "check-workflow-policy.mjs"), + "retrieval-sidecar-smoke.yml", + ], + [ + path.join(repoRoot, ".github", "scripts", "route-ci-proof.mjs"), + ".github/workflows/retrieval-sidecar-smoke.yml", + ], +].map(([filePath, includes]) => ({ + filePath: path.resolve(filePath), includes, })); @@ -1508,10 +1522,11 @@ function scanCorpusHarnessDependencies(prepared) { const normalizedLine = normalizeNativeSeparators(line.text, line.shellLike); const hasProtectedDependency = corpusHarnessDependencyRegexes.some((pattern) => pattern.test(normalizedLine) - && !allowedHarnessDependencyLines.some(({ filePath, includes }) => - path.resolve(prepared.filePath) === filePath - && line.text.includes(includes) - && pattern.test(includes) + && !harnessDependencyAllowed( + prepared, + line.startLine, + line.endLine, + (includes) => pattern.test(normalizeNativeSeparators(includes)), ) ); if (hasProtectedDependency) { @@ -1523,6 +1538,26 @@ function scanCorpusHarnessDependencies(prepared) { return hits; } +function harnessDependencyAllowed(prepared, startLine, endLine, markerMatches) { + const source = prepared.sourceLines.slice(startLine - 1, endLine).join("\n"); + return allowedHarnessReferences.some(({ filePath, includes }) => + path.resolve(prepared.filePath) === filePath + && source.includes(includes) + && markerMatches(includes) + ); +} + +function compactHarnessDependencyAllowed(prepared, literals, marker) { + return allowedHarnessReferences.some(({ filePath, includes }) => { + const includesCompact = compactProductionSource(normalizeNativeSeparators(includes)); + return path.resolve(prepared.filePath) === filePath + && literals.some(({ literal }) => + normalizeNativeSeparators(staticStringLiteralContent(literal)).includes(includes) + ) + && (marker.includes(includesCompact) || includesCompact.includes(marker)); + }); +} + function scanProductionFile(prepared, patterns, combinedRe) { const lines = prepared.logicalLines ?? prepared.lines.map((text, index) => ({ text, @@ -1619,6 +1654,7 @@ function scanProductionCompactPatterns( marker, minimumLiteralCount = 1, allowSurroundingText = false, + matchAllowed = () => false, ) { const production = prepared.staticSource ?? prepared.production; const markerLower = marker.toLowerCase(); @@ -1641,14 +1677,21 @@ function scanProductionCompactPatterns( ) { break; } + const latestBoundary = compact.length; compact += compactProductionSource(literals[end].literal); - const containsMarker = allowSurroundingText - ? compact.includes(markerLower) - : compact === markerLower; - if (end - start + 1 >= minimumLiteralCount && containsMarker) { - hits.push( - compactPatternHit(prepared.filePath, literals[start].line, literals[end].line, marker), + const matchedRequiredLiterals = end - start + 1 >= minimumLiteralCount + && compactMarkerMatches( + compact, + markerLower, + allowSurroundingText, + minimumLiteralCount > 1 ? latestBoundary : null, ); + if (matchedRequiredLiterals) { + if (!matchAllowed(literals.slice(start, end + 1), markerLower)) { + hits.push( + compactPatternHit(prepared.filePath, literals[start].line, literals[end].line, marker), + ); + } break; } if (!allowSurroundingText && compact.length >= markerLower.length) { @@ -1659,6 +1702,29 @@ function scanProductionCompactPatterns( return hits; } +function compactMarkerMatches(compact, marker, allowSurroundingText, requiredBoundary) { + if (!allowSurroundingText) { + return compact === marker + && ( + requiredBoundary == null + || (requiredBoundary > 0 && marker.length > requiredBoundary) + ); + } + for ( + let offset = compact.indexOf(marker); + offset >= 0; + offset = compact.indexOf(marker, offset + 1) + ) { + if ( + requiredBoundary == null + || (offset < requiredBoundary && offset + marker.length > requiredBoundary) + ) { + return true; + } + } + return false; +} + function staticStringLiteralSpans(text) { const literals = []; const lineStarts = [0]; @@ -1907,7 +1973,13 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { } } for (const pattern of corpusHarnessCompactPatternList) { - const hits = scanProductionCompactPatterns(prepared, pattern, 2, true); + const hits = scanProductionCompactPatterns( + prepared, + pattern, + 2, + true, + (literals, marker) => compactHarnessDependencyAllowed(prepared, literals, marker), + ); if (hits.length > 0) { console.error( `Constructed evaluation/proof harness dependency /${pattern}/ in protected non-Rust path ${path.relative(repoRoot, filePath)}:\n${hits.join("\n")}\n`, From 0cd9b9ae3a84c74cdb56190bfca31660c7b7bfb7 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Mon, 13 Jul 2026 16:21:55 -0400 Subject: [PATCH 11/11] bind corpus guard allowances --- CHANGELOG.md | 5 +- .../tests/retrieval_generalization_guard.rs | 56 +++++++- scripts/lint-retrieval-generalization.mjs | 125 +++++++++++++----- 3 files changed, 147 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bea1a94..00f6bacd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,8 +45,9 @@ the full protected inventory, including local composite actions and shipped agent instruction surfaces, and missing protected paths fail closed. Exact release-policy references to the retrieval smoke workflow are allowed only - by file and literal, while constructed dependencies must span the literals - that form the matched harness marker. + by file, literal, and use; every direct or constructed occurrence is checked, + and constructed dependencies must span the literals that form the matched + harness marker. - Added deterministic prior-version managed CLI upgrade proof to every native managed-lifecycle cell. The gate now starts with a verified older install, requires the requested packaged binary to serve status and grounding, and diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index cfe09433..7b3bd04b 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -511,9 +511,18 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { ("quoted-yaml-scalar.yml", "fetch-holdout-repos.mjs"), ("quoted-block-scalar.yml", "fetch-holdout-repos.mjs"), ("github-script.yml", "fetchholdoutreposmjs"), - ("direct-harness-import.mjs", "codestory-agent-ab-benchmark.mjs"), - ("constructed-harness-import.mjs", "codestoryagentabbenchmarkmjs"), - ("unapproved-policy-reference.mjs", "retrieval-sidecar-smoke.yml"), + ( + "direct-harness-import.mjs", + "codestory-agent-ab-benchmark.mjs", + ), + ( + "constructed-harness-import.mjs", + "codestoryagentabbenchmarkmjs", + ), + ( + "unapproved-policy-reference.mjs", + "retrieval-sidecar-smoke.yml", + ), ("skill.md", "fetch-holdout-repos.mjs"), ("codestory.mdc", "benchmarks/tasks"), ] { @@ -580,6 +589,47 @@ fn linter_rejects_direct_and_split_non_rust_corpus_dependencies() { ); } +#[test] +fn linter_binds_policy_allowances_to_the_exact_approved_use() { + let allowed = run_lint_with_non_rust_fixtures(&[ + ( + ".github/scripts/route-ci-proof.mjs", + " \".github/workflows/retrieval-sidecar-smoke.yml\",\n", + ), + ( + ".github/scripts/check-workflow-policy.mjs", + "const retrievalSidecarSmoke = path.join(workflowRoot, \"retrieval-sidecar-smoke.yml\");\nviolations.push(\"retrieval-sidecar-smoke.yml must exist\");\nviolations.push(\"retrieval-sidecar-smoke.yml Windows proof must be workflow_dispatch-only\");\n", + ), + ]); + let allowed_stderr = String::from_utf8_lossy(&allowed.stderr); + assert!( + allowed.status.success(), + "the exact policy and routing references must pass lint; stderr={allowed_stderr}" + ); + + let rejected = run_lint_with_non_rust_fixtures(&[ + ( + ".github/scripts/route-ci-proof.mjs", + " \".github/workflows/retrieval-sidecar-smoke.yml\",\n \".github/workflows/retrieval-sidecar-smoke.yml\",\nawait import(\".github/workflows/retrieval-sidecar-smoke.yml\");\n", + ), + ( + ".github/scripts/check-workflow-policy.mjs", + "const retrievalSidecarSmoke = path.join(workflowRoot, \"retrieval-sidecar-smoke.yml\");\nconst hostile = \".github/workflows/retrieval-\" + \"sidecar-smoke.yml\";\n", + ), + ]); + let rejected_stderr = String::from_utf8_lossy(&rejected.stderr).to_ascii_lowercase(); + assert!( + !rejected.status.success(), + "an approved file must not hide another harness use; stderr={rejected_stderr}" + ); + assert!( + rejected_stderr.contains("route-ci-proof.mjs:3:await import") + && rejected_stderr.contains("check-workflow-policy.mjs:2:") + && rejected_stderr.contains("githubworkflowsretrievalsidecarsmokeyml"), + "lint must identify both the hostile direct and split uses; stderr={rejected_stderr}" + ); +} + #[test] fn linter_fails_closed_when_one_prompt_corpus_entry_is_not_a_literal() { let output = run_lint_with_prompt_script_fixture( diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index c5a2c2cf..9cf5a2f1 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -91,24 +91,39 @@ const corpusSupportNonRustFiles = new Set([ ].map((filePath) => path.resolve(filePath))); const allowedHarnessReferences = [ [ - path.join(groundingSkillDir, "references", "drill-suite.md"), + path.join("plugins", "codestory", "skills", "codestory-grounding", "references", "drill-suite.md"), "scripts/score-drill-ledger.mjs", + "`node scripts/score-drill-ledger.mjs [scored-report.json]`.", ], [ - path.join(groundingSkillDir, "references", "retrieval-rollout.md"), + path.join("plugins", "codestory", "skills", "codestory-grounding", "references", "retrieval-rollout.md"), ".github/workflows/retrieval-sidecar-smoke.yml", + "| Smoke CI | `.github/workflows/retrieval-sidecar-smoke.yml` plus `docs/ops/retrieval-sidecars.md#preflight-smoke-contract` pass criteria | PRs touching retrieval crate, runtime/stdio/search wiring, indexer retrieval hooks, retrieval docs, scripts, Docker sidecar config, or the workflow | Full sidecar readiness. CI smoke uses `--skip-compose --wait-secs 0` and proves manifest-missing fail-closed shape only |", ], [ - path.join(repoRoot, ".github", "scripts", "check-workflow-policy.mjs"), + path.join(".github", "scripts", "check-workflow-policy.mjs"), "retrieval-sidecar-smoke.yml", + "const retrievalSidecarSmoke = path.join(workflowRoot, \"retrieval-sidecar-smoke.yml\");", ], [ - path.join(repoRoot, ".github", "scripts", "route-ci-proof.mjs"), + path.join(".github", "scripts", "check-workflow-policy.mjs"), + "retrieval-sidecar-smoke.yml", + "violations.push(\"retrieval-sidecar-smoke.yml must exist\");", + ], + [ + path.join(".github", "scripts", "check-workflow-policy.mjs"), + "retrieval-sidecar-smoke.yml", + "violations.push(\"retrieval-sidecar-smoke.yml Windows proof must be workflow_dispatch-only\");", + ], + [ + path.join(".github", "scripts", "route-ci-proof.mjs"), ".github/workflows/retrieval-sidecar-smoke.yml", + "\".github/workflows/retrieval-sidecar-smoke.yml\",", ], -].map(([filePath, includes]) => ({ - filePath: path.resolve(filePath), +].map(([relativePath, includes, use]) => ({ + relativePath, includes, + use, })); const executableJavaScriptExtensions = new Set([ @@ -1519,14 +1534,10 @@ function openQuoteAtEnd(text, escape) { function scanCorpusHarnessDependencies(prepared) { const hits = []; for (const line of prepared.logicalLines) { - const normalizedLine = normalizeNativeSeparators(line.text, line.shellLike); + const normalizedLine = normalizeNativeSeparators(line.text, line.shellLike).trim(); const hasProtectedDependency = corpusHarnessDependencyRegexes.some((pattern) => - pattern.test(normalizedLine) - && !harnessDependencyAllowed( - prepared, - line.startLine, - line.endLine, - (includes) => pattern.test(normalizeNativeSeparators(includes)), + regexOccurrences(pattern, normalizedLine).some((occurrence) => + !harnessDependencyAllowed(prepared, line, pattern, occurrence) ) ); if (hasProtectedDependency) { @@ -1538,19 +1549,34 @@ function scanCorpusHarnessDependencies(prepared) { return hits; } -function harnessDependencyAllowed(prepared, startLine, endLine, markerMatches) { - const source = prepared.sourceLines.slice(startLine - 1, endLine).join("\n"); - return allowedHarnessReferences.some(({ filePath, includes }) => - path.resolve(prepared.filePath) === filePath - && source.includes(includes) - && markerMatches(includes) +function regexOccurrences(pattern, source) { + const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`; + return [...source.matchAll(new RegExp(pattern.source, flags))].map((match) => ({ + index: match.index, + length: match[0].length, + })); +} + +function harnessDependencyAllowed(prepared, line, pattern, occurrence) { + return allowedHarnessReferences.some(({ relativePath, includes, use }) => + allowedReferenceFileMatches(prepared.filePath, relativePath) + && allowedReferenceUseMatches(prepared, use, line.startLine, line.endLine) + && regexOccurrences(pattern, normalizeNativeSeparators(use)).some((allowed) => + allowed.index === occurrence.index && allowed.length === occurrence.length + ) + && pattern.test(normalizeNativeSeparators(includes)) ); } function compactHarnessDependencyAllowed(prepared, literals, marker) { - return allowedHarnessReferences.some(({ filePath, includes }) => { + const startLine = Math.min(...literals.map(({ line }) => line)); + const endLine = Math.max(...literals.map(({ line }) => line)); + const source = prepared.sourceLines.slice(startLine - 1, endLine).join("\n").trim(); + return allowedHarnessReferences.some(({ relativePath, includes, use }) => { const includesCompact = compactProductionSource(normalizeNativeSeparators(includes)); - return path.resolve(prepared.filePath) === filePath + return allowedReferenceFileMatches(prepared.filePath, relativePath) + && source === use + && allowedReferenceUseMatches(prepared, use, startLine, endLine) && literals.some(({ literal }) => normalizeNativeSeparators(staticStringLiteralContent(literal)).includes(includes) ) @@ -1558,6 +1584,22 @@ function compactHarnessDependencyAllowed(prepared, literals, marker) { }); } +function allowedReferenceFileMatches(filePath, relativePath) { + const roots = usesDefaultNonRustScanRoots ? [repoRoot] : nonRustScanRoots; + return roots.some((root) => + path.resolve(filePath) === path.resolve(root, relativePath) + ); +} + +function allowedReferenceUseMatches(prepared, use, startLine, endLine) { + const matches = prepared.logicalLines.filter((line) => + normalizeNativeSeparators(line.text, line.shellLike).trim() === use + ); + return matches.length === 1 + && matches[0].startLine === startLine + && matches[0].endLine === endLine; +} + function scanProductionFile(prepared, patterns, combinedRe) { const lines = prepared.logicalLines ?? prepared.lines.map((text, index) => ({ text, @@ -1665,6 +1707,7 @@ function scanProductionCompactPatterns( const literals = prepared.literals; for (let start = 0; start < literals.length; start += 1) { let compact = ""; + const segments = []; for (let end = start; end < literals.length; end += 1) { if ( end > start @@ -1678,20 +1721,31 @@ function scanProductionCompactPatterns( break; } const latestBoundary = compact.length; - compact += compactProductionSource(literals[end].literal); - const matchedRequiredLiterals = end - start + 1 >= minimumLiteralCount - && compactMarkerMatches( + const literalCompact = compactProductionSource(literals[end].literal); + compact += literalCompact; + segments.push({ + ...literals[end], + compactStart: latestBoundary, + compactEnd: compact.length, + }); + const occurrences = end - start + 1 >= minimumLiteralCount + ? compactMarkerOccurrences( compact, markerLower, allowSurroundingText, minimumLiteralCount > 1 ? latestBoundary : null, + ) + : []; + const rejected = occurrences.some(({ index, length }) => { + const matchedLiterals = segments.filter((literal) => + literal.compactStart < index + length && literal.compactEnd > index + ); + return !matchAllowed(matchedLiterals, markerLower); + }); + if (rejected) { + hits.push( + compactPatternHit(prepared.filePath, literals[start].line, literals[end].line, marker), ); - if (matchedRequiredLiterals) { - if (!matchAllowed(literals.slice(start, end + 1), markerLower)) { - hits.push( - compactPatternHit(prepared.filePath, literals[start].line, literals[end].line, marker), - ); - } break; } if (!allowSurroundingText && compact.length >= markerLower.length) { @@ -1702,14 +1756,17 @@ function scanProductionCompactPatterns( return hits; } -function compactMarkerMatches(compact, marker, allowSurroundingText, requiredBoundary) { +function compactMarkerOccurrences(compact, marker, allowSurroundingText, requiredBoundary) { if (!allowSurroundingText) { return compact === marker && ( requiredBoundary == null || (requiredBoundary > 0 && marker.length > requiredBoundary) - ); + ) + ? [{ index: 0, length: marker.length }] + : []; } + const occurrences = []; for ( let offset = compact.indexOf(marker); offset >= 0; @@ -1719,10 +1776,10 @@ function compactMarkerMatches(compact, marker, allowSurroundingText, requiredBou requiredBoundary == null || (offset < requiredBoundary && offset + marker.length > requiredBoundary) ) { - return true; + occurrences.push({ index: offset, length: marker.length }); } } - return false; + return occurrences; } function staticStringLiteralSpans(text) {