Add installed-plugin proof infrastructure

docs/benchmark-method.md

@@ -39,6 +39,9 @@ answer does not prove the agent can execute the loop.
   gains justify the increase.
 - Use at least three samples per case before making a public numerical claim.
 - At least three fixture-repository trajectories pass their independent proof.
+- For a version-to-version sealed regression gate, pass the frozen prior skill
+  with `--baseline-skill-file`; the generic no-skill control is not a substitute
+  for the released baseline.
 
 ## Anti-Overfit Rule
 

docs/installed-plugin-probe.md

@@ -0,0 +1,46 @@
+# Installed Plugin Access Probe
+
+This probe answers one narrow question: does Claude Code expose trustworthy raw
+evidence that an installed LoopSpine plugin session read an exact file inside
+the selected plugin root?
+
+Run it from the repository root:
+
+```bash
+npm run probe:installed-plugin
+```
+
+The default probe invokes `/loopspine:loopspine` through Claude Code's
+`--plugin-dir` path and explicitly asks the `Read` tool to open
+`docs/design.md`. A different file inside the plugin root can be selected with:
+
+```bash
+npm run probe:installed-plugin -- --reference path/to/reference.md
+```
+
+## Passing Evidence
+
+The probe fails unless the Claude stream contains all of the following in one
+session and in order:
+
+1. An `init` event identifying the exact LoopSpine plugin root, plan permission
+   mode, no MCP servers, and `Read` as the only available tool.
+2. A `Read` `tool_use` event whose real path equals the expected file.
+3. A correlated `tool_result` with the same tool-use ID, exact real path, and
+   content matching the expected SHA-256.
+4. A successful terminal `result` after the tool result.
+
+It also requires the reference hash and tracked worktree status to remain
+unchanged. Model prose such as "I read the reference" is never accepted as
+evidence.
+
+Raw stream output, stderr, and the compact receipt are written under
+`results/probes/<timestamp>/`. Local probe output is ignored by Git until a
+specific receipt is deliberately promoted.
+
+## Proof Boundary
+
+A passing probe proves that Claude Code's installed-plugin stream exposes an
+auditable file-access event. It does not prove that LoopSpine automatically
+chooses an adaptive reference, that tiny tasks avoid it, or that the reference
+improves task outcomes. Those remain separate candidate-v3 gates.

package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "description": "A benchmarked development loop for coding agents.",
   "scripts": {
-    "test": "node scripts/validate.mjs && node scripts/test-matching.mjs && node scripts/test-dogfood-metrics.mjs",
+    "test": "node scripts/validate.mjs && node scripts/test-matching.mjs && node scripts/test-dogfood-metrics.mjs && node scripts/test-claude-access-events.mjs",
     "benchmark:pilot": "node scripts/run-benchmark.mjs --pilot",
     "benchmark": "node scripts/run-benchmark.mjs",
     "benchmark:sealed": "node scripts/run-benchmark.mjs --sealed-only --sealed-file evals/sealed-v2.json --samples 3 --seed loopspine-v2",
@@ -17,7 +17,8 @@
     "demo": "node scripts/run-demo.mjs",
     "demo:render": "node scripts/render-demo.mjs",
     "dogfood:report": "node scripts/dogfood-report.mjs --write",
-    "dogfood:record": "node scripts/record-dogfood.mjs"
+    "dogfood:record": "node scripts/record-dogfood.mjs",
+    "probe:installed-plugin": "node scripts/run-installed-plugin-probe.mjs"
   },
   "engines": {
     "node": ">=20"

scripts/claude-access-events.mjs

@@ -0,0 +1,184 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+function sha256(value) {
+  return crypto.createHash("sha256").update(value).digest("hex");
+}
+
+function canonicalPath(filePath, label) {
+  try {
+    return fs.realpathSync(filePath);
+  } catch (error) {
+    throw new Error(`${label} does not resolve: ${error.message}`);
+  }
+}
+
+function isInside(root, target) {
+  const relative = path.relative(root, target);
+  return relative === "" || (!path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`));
+}
+
+export function parseClaudeStream(rawOutput) {
+  const lines = String(rawOutput).split(/\r?\n/);
+  const events = [];
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index].trim();
+    if (!line) continue;
+    try {
+      const event = JSON.parse(line);
+      if (!event || typeof event !== "object" || Array.isArray(event)) throw new Error("event is not an object");
+      events.push({ event, line: index + 1 });
+    } catch (error) {
+      throw new Error(`malformed Claude stream event on line ${index + 1}: ${error.message}`);
+    }
+  }
+  if (!events.length) throw new Error("Claude stream contains no events");
+  return events;
+}
+
+function verifyClaudeSession(events, canonicalRoot) {
+  const sessionIds = new Set(events.map((event) => event.session_id).filter((value) => typeof value === "string" && value));
+  if (sessionIds.size !== 1) throw new Error("Claude stream must contain exactly one session id");
+  const [sessionId] = sessionIds;
+
+  const initIndex = events.findIndex((event) => event.type === "system" && event.subtype === "init");
+  if (initIndex < 0) throw new Error("Claude stream is missing the init event");
+  const init = events[initIndex];
+  if (!Array.isArray(init.tools) || init.tools.length !== 1 || init.tools[0] !== "Read") {
+    throw new Error("Claude init event is not restricted to the Read tool");
+  }
+  if (init.permissionMode !== "plan") throw new Error("Claude init event is not in plan permission mode");
+  if (!Array.isArray(init.mcp_servers) || init.mcp_servers.length !== 0) {
+    throw new Error("Claude init event exposes MCP servers during the read-only probe");
+  }
+  const pluginLoaded = Array.isArray(init.plugins) && init.plugins.some((plugin) => {
+    if (plugin?.name !== "loopspine" || typeof plugin.path !== "string") return false;
+    try {
+      return canonicalPath(plugin.path, "loaded plugin path") === canonicalRoot;
+    } catch {
+      return false;
+    }
+  });
+  if (!pluginLoaded) throw new Error("Claude init event does not identify the expected LoopSpine plugin root");
+
+  const resultIndex = events.findIndex((event) => event.type === "result");
+  if (resultIndex < 0) throw new Error("Claude stream is missing the terminal result event");
+  if (initIndex >= resultIndex) throw new Error("Claude init event must occur before the terminal result");
+  const result = events[resultIndex];
+  if (result.subtype !== "success" || result.is_error) throw new Error("Claude terminal result is not successful");
+  return { init, initIndex, resultIndex, sessionId };
+}
+
+export function verifyClaudeNoReadEvent({ rawOutput, pluginRoot, referencePath }) {
+  const canonicalRoot = canonicalPath(pluginRoot, "plugin root");
+  const canonicalReference = canonicalPath(referencePath, "reference file");
+  if (!isInside(canonicalRoot, canonicalReference) || canonicalReference === canonicalRoot) {
+    throw new Error("reference file must resolve inside the plugin root");
+  }
+  const events = parseClaudeStream(rawOutput).map(({ event }) => event);
+  const session = verifyClaudeSession(events, canonicalRoot);
+  let referenceReadEvents = 0;
+  for (let eventIndex = 0; eventIndex < session.resultIndex; eventIndex += 1) {
+    const blocks = Array.isArray(events[eventIndex].message?.content) ? events[eventIndex].message.content : [];
+    for (const block of blocks) {
+      if (block?.type !== "tool_use" || block.name !== "Read" || typeof block.input?.file_path !== "string") continue;
+      try {
+        if (canonicalPath(block.input.file_path, "Read tool path") === canonicalReference) referenceReadEvents += 1;
+      } catch {
+        // An unrelated failed path is not evidence that the reference was read.
+      }
+    }
+  }
+  if (referenceReadEvents) throw new Error("Claude stream contains an unexpected Read tool_use event for the reference file");
+  return {
+    session_id: session.sessionId,
+    plugin_root: canonicalRoot,
+    reference_path: canonicalReference,
+    reference_read_events: 0,
+    init_event_index: session.initIndex,
+    result_event_index: session.resultIndex,
+    available_tools: session.init.tools,
+    mcp_servers: session.init.mcp_servers,
+    permission_mode: session.init.permissionMode,
+    claude_code_version: session.init.claude_code_version || null,
+    model: session.init.model || null
+  };
+}
+
+export function verifyClaudeReadEvent({ rawOutput, pluginRoot, referencePath, referenceSha256 }) {
+  const canonicalRoot = canonicalPath(pluginRoot, "plugin root");
+  const canonicalReference = canonicalPath(referencePath, "reference file");
+  if (!isInside(canonicalRoot, canonicalReference) || canonicalReference === canonicalRoot) {
+    throw new Error("reference file must resolve inside the plugin root");
+  }
+
+  const actualReferenceContent = fs.readFileSync(canonicalReference);
+  const actualReferenceSha256 = sha256(actualReferenceContent);
+  if (actualReferenceSha256 !== referenceSha256) {
+    throw new Error("reference file hash does not match the expected hash");
+  }
+
+  const parsed = parseClaudeStream(rawOutput);
+  const events = parsed.map(({ event }) => event);
+  const session = verifyClaudeSession(events, canonicalRoot);
+  const { init, initIndex, resultIndex, sessionId } = session;
+
+  let matchedToolUse = null;
+  for (let eventIndex = initIndex + 1; eventIndex < resultIndex; eventIndex += 1) {
+    const event = events[eventIndex];
+    if (event.type !== "assistant" || !Array.isArray(event.message?.content)) continue;
+    for (const block of event.message.content) {
+      if (block?.type !== "tool_use" || block.name !== "Read" || typeof block.id !== "string" || typeof block.input?.file_path !== "string") continue;
+      let toolPath;
+      try {
+        toolPath = canonicalPath(block.input.file_path, "Read tool path");
+      } catch {
+        continue;
+      }
+      if (toolPath === canonicalReference) {
+        matchedToolUse = { eventIndex, id: block.id };
+        break;
+      }
+    }
+    if (matchedToolUse) break;
+  }
+  if (!matchedToolUse) throw new Error("Claude stream has no matching Read tool_use event before the terminal result");
+
+  let matchedToolResult = null;
+  for (let eventIndex = matchedToolUse.eventIndex + 1; eventIndex < events.length; eventIndex += 1) {
+    const event = events[eventIndex];
+    const blocks = Array.isArray(event.message?.content) ? event.message.content : [];
+    const correlated = blocks.some((block) => block?.type === "tool_result" && block.tool_use_id === matchedToolUse.id);
+    if (!correlated) continue;
+    if (eventIndex >= resultIndex) throw new Error("correlated tool_result must occur before the terminal result");
+
+    const file = event.tool_use_result?.file;
+    if (!file || typeof file.filePath !== "string" || typeof file.content !== "string") {
+      throw new Error("correlated tool_result is missing structured file evidence");
+    }
+    const resultPath = canonicalPath(file.filePath, "tool result file path");
+    if (resultPath !== canonicalReference) throw new Error("correlated tool_result names the wrong file");
+    if (sha256(file.content) !== referenceSha256) throw new Error("correlated tool_result content hash does not match the reference");
+    matchedToolResult = { eventIndex };
+    break;
+  }
+  if (!matchedToolResult) throw new Error("Claude stream has no correlated tool_result for the matching Read event");
+
+  return {
+    session_id: sessionId,
+    plugin_root: canonicalRoot,
+    reference_path: canonicalReference,
+    reference_sha256: referenceSha256,
+    init_event_index: initIndex,
+    tool_use_event_index: matchedToolUse.eventIndex,
+    tool_result_event_index: matchedToolResult.eventIndex,
+    result_event_index: resultIndex,
+    tool_use_id: matchedToolUse.id,
+    available_tools: init.tools,
+    mcp_servers: init.mcp_servers,
+    permission_mode: init.permissionMode,
+    claude_code_version: init.claude_code_version || null,
+    model: init.model || null
+  };
+}

scripts/run-benchmark.mjs

@@ -9,7 +9,7 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
 const variants = ["without-skill", "with-skill"];
 
 function usage() {
-  console.error("Usage: node run-benchmark.mjs [--pilot] [--sealed|--sealed-only] [--sealed-file PATH] [--samples N] [--seed VALUE] [--model NAME]");
+  console.error("Usage: node run-benchmark.mjs [--pilot] [--sealed|--sealed-only] [--sealed-file PATH] [--baseline-skill-file PATH] [--samples N] [--seed VALUE] [--model NAME]");
 }
 
 function fail(message, code = 2) {
@@ -18,7 +18,7 @@ function fail(message, code = 2) {
 }
 
 function parseArgs(argv) {
-  const options = { pilot: false, sealed: false, sealedOnly: false, sealedFile: path.join("evals", "sealed-v2.json"), samples: null, seed: "loopspine-v2", model: process.env.LOOPSPINE_MODEL || "gpt-5.5" };
+  const options = { pilot: false, sealed: false, sealedOnly: false, sealedFile: path.join("evals", "sealed-v2.json"), baselineSkillFile: null, samples: null, seed: "loopspine-v2", model: process.env.LOOPSPINE_MODEL || "gpt-5.5" };
   for (let index = 0; index < argv.length; index += 1) {
     const arg = argv[index];
     if (arg === "--help") {
@@ -28,14 +28,15 @@ function parseArgs(argv) {
     if (arg === "--pilot") options.pilot = true;
     else if (arg === "--sealed") options.sealed = true;
     else if (arg === "--sealed-only") options.sealedOnly = true;
-    else if (arg === "--samples" || arg === "--seed" || arg === "--model" || arg === "--sealed-file") {
+    else if (arg === "--samples" || arg === "--seed" || arg === "--model" || arg === "--sealed-file" || arg === "--baseline-skill-file") {
       const value = argv[index + 1];
       if (!value || value.startsWith("--")) fail(`Missing value for ${arg}`);
       if (arg === "--samples") {
         if (!/^\d+$/.test(value) || Number(value) < 1) fail("--samples must be a positive integer");
         options.samples = Number(value);
       } else if (arg === "--seed") options.seed = value;
       else if (arg === "--sealed-file") options.sealedFile = value;
+      else if (arg === "--baseline-skill-file") options.baselineSkillFile = value;
       else options.model = value;
       index += 1;
     } else {
@@ -125,6 +126,18 @@ try {
 } catch (error) {
   fail(`Missing execution evidence: ${skillPath}: ${error.message}`);
 }
+let baselineInstructions = "You are a capable senior coding agent. Choose a sensible workflow for the software task. Preserve ordinary safety boundaries and do not execute the scenario.";
+let baselineSkillPath = null;
+if (options.baselineSkillFile) {
+  baselineSkillPath = path.isAbsolute(options.baselineSkillFile)
+    ? options.baselineSkillFile
+    : path.resolve(root, options.baselineSkillFile);
+  try {
+    baselineInstructions = fs.readFileSync(baselineSkillPath, "utf8");
+  } catch (error) {
+    fail(`Missing baseline skill evidence: ${baselineSkillPath}: ${error.message}`);
+  }
+}
 
 const startedAt = new Date().toISOString();
 const stamp = startedAt.replace(/[:.]/g, "-");
@@ -141,6 +154,7 @@ const provenance = {
   sealed: options.sealed || options.sealedOnly,
   sealed_only: options.sealedOnly,
   skill_sha256: hashFile(skillPath),
+  baseline_skill: baselineSkillPath ? { path: baselineSkillPath, sha256: hashFile(baselineSkillPath) } : null,
   eval_files: [development, ...(sealed ? [sealed] : [])].map(({ relativePath, sha256, source }) => ({ path: relativePath, source, sha256 })),
   command_args: process.argv.slice(2),
   reasoning_effort: process.env.LOOPSPINE_REASONING_EFFORT || "provider-default",
@@ -155,7 +169,6 @@ const provenance = {
 };
 fs.writeFileSync(path.join(runDir, "provenance.json"), `${JSON.stringify(provenance, null, 2)}\n`);
 
-const baseline = "You are a capable senior coding agent. Choose a sensible workflow for the software task. Preserve ordinary safety boundaries and do not execute the scenario.";
 const timings = [];
 const random = createRandom(options.seed);
 
@@ -166,7 +179,7 @@ for (const item of cases) {
       const outputDir = path.join(runDir, variant, item.id);
       const outputPath = path.join(outputDir, `sample-${sample}.txt`);
       fs.mkdirSync(outputDir, { recursive: true });
-      const instructions = variant === "with-skill" ? skill : baseline;
+      const instructions = variant === "with-skill" ? skill : baselineInstructions;
       const prompt = `${instructions}\n\n# Scenario\n${item.prompt}\n\nExplain the workflow you would follow. Do not execute commands or edit files.`;
       const started = Date.now();
       console.log(`[${variant}] ${item.id} sample ${sample}/${options.samples}`);