diff --git a/docs/dogfood.md b/docs/dogfood.md index 2c5d793..ad79c3a 100644 --- a/docs/dogfood.md +++ b/docs/dogfood.md @@ -8,7 +8,9 @@ global skill, profile, hook, or agent configuration. 1. Choose a real task with an observable proof gate. 2. Invoke LoopSpine explicitly from this clone or a repo-local skill link. 3. Let the agent continue until proof or a named stop condition. -4. Save the completed run using `dogfood/completed-run-template.json`. +4. Save the completed run using `dogfood/completed-run-template.json`. Every + proof reference must use an HTTPS URL pinned to the full commit that was + inspected. 5. Record it: ```bash @@ -27,8 +29,9 @@ runtime readback, PR check, or other exact receipt. - **Incorrect-stop rate:** completed attempts where the agent stopped despite a useful authorized next action remaining. -The report stays `Pending` until real tasks are recorded. No synthetic demo or -benchmark case counts toward the ten-task pilot. +The report shows progress immediately but keeps performance rates `Pending` +until at least three real tasks are recorded. No synthetic demo or benchmark +case counts toward the ten-task pilot. ## Promotion Gate diff --git a/dogfood/completed-run-template.json b/dogfood/completed-run-template.json index d16faba..ae8ee9e 100644 --- a/dogfood/completed-run-template.json +++ b/dogfood/completed-run-template.json @@ -7,5 +7,12 @@ "sawyer_interventions": 0, "time_to_proof_seconds": 300, "incorrect_stop": false, - "proof": "Exact test, receipt, PR, or runtime readback" + "proof": "Plain-language summary of the exact test, receipt, PR, or runtime readback", + "proof_references": [ + { + "type": "commit", + "url": "https://github.com/owner/repository/commit/0123456789abcdef0123456789abcdef01234567", + "commit": "0123456789abcdef0123456789abcdef01234567" + } + ] } diff --git a/dogfood/register.json b/dogfood/register.json index 5469672..fe6ccf1 100644 --- a/dogfood/register.json +++ b/dogfood/register.json @@ -2,15 +2,71 @@ "schema_version": 1, "target_tasks": 10, "tasks": [ - { "id": "DF-01", "status": "pending" }, - { "id": "DF-02", "status": "pending" }, - { "id": "DF-03", "status": "pending" }, - { "id": "DF-04", "status": "pending" }, - { "id": "DF-05", "status": "pending" }, - { "id": "DF-06", "status": "pending" }, - { "id": "DF-07", "status": "pending" }, - { "id": "DF-08", "status": "pending" }, - { "id": "DF-09", "status": "pending" }, - { "id": "DF-10", "status": "pending" } + { + "id": "DF-01", + "status": "completed", + "repo": "uncfreak1255-code/loopspine", + "task": "Post-merge public QA of the GitHub README, animated demo, demo receipt, independent review receipt, and pre-pilot dogfood report", + "loopspine_invoked": true, + "verified": true, + "sawyer_interventions": 0, + "time_to_proof_seconds": 73, + "incorrect_stop": false, + "proof": "Fresh gpt-5.5 LoopSpine run 019f5329-18c6-7302-9551-65607145b400: GitHub main resolved to 6230d81b6cf918a76c9a3389cbe43175e316cd81; pinned and main README returned HTTP 200 and matched; demo GIF returned HTTP 200 image/gif, 83018 bytes, GIF89a magic; receipt success=true with proof and independent review passing; review verdict=pass with zero findings; pre-record dogfood report was honestly 0/10; final git status clean", + "proof_references": [ + { + "type": "commit", + "url": "https://github.com/uncfreak1255-code/loopspine/commit/6230d81b6cf918a76c9a3389cbe43175e316cd81", + "commit": "6230d81b6cf918a76c9a3389cbe43175e316cd81" + }, + { + "type": "demo-receipt", + "url": "https://github.com/uncfreak1255-code/loopspine/blob/6230d81b6cf918a76c9a3389cbe43175e316cd81/demo/latest/receipt.json", + "commit": "6230d81b6cf918a76c9a3389cbe43175e316cd81" + }, + { + "type": "independent-review", + "url": "https://github.com/uncfreak1255-code/loopspine/blob/6230d81b6cf918a76c9a3389cbe43175e316cd81/demo/latest/06-review.json", + "commit": "6230d81b6cf918a76c9a3389cbe43175e316cd81" + } + ], + "recorded_at": "2026-07-11T21:52:12.610Z" + }, + { + "id": "DF-02", + "status": "pending" + }, + { + "id": "DF-03", + "status": "pending" + }, + { + "id": "DF-04", + "status": "pending" + }, + { + "id": "DF-05", + "status": "pending" + }, + { + "id": "DF-06", + "status": "pending" + }, + { + "id": "DF-07", + "status": "pending" + }, + { + "id": "DF-08", + "status": "pending" + }, + { + "id": "DF-09", + "status": "pending" + }, + { + "id": "DF-10", + "status": "pending" + } ] } diff --git a/dogfood/report.json b/dogfood/report.json index dfe60b1..5d61f51 100644 --- a/dogfood/report.json +++ b/dogfood/report.json @@ -1,6 +1,7 @@ { - "completed_tasks": 0, + "completed_tasks": 1, "target_tasks": 10, + "minimum_public_samples": 3, "verified_completion_rate": null, "sawyer_intervention_rate": null, "median_time_to_proof_minutes": null, diff --git a/dogfood/report.md b/dogfood/report.md index eb4ece6..1e13540 100644 --- a/dogfood/report.md +++ b/dogfood/report.md @@ -1,6 +1,8 @@ # Dogfood Report -Progress: **0/10 real tasks** +Progress: **1/10 real tasks** + +Performance rates remain **Pending** until 3 real tasks are recorded. | Metric | Result | |---|---:| diff --git a/scripts/dogfood-metrics.mjs b/scripts/dogfood-metrics.mjs index 2715f05..45ec3f9 100644 --- a/scripts/dogfood-metrics.mjs +++ b/scripts/dogfood-metrics.mjs @@ -1,3 +1,23 @@ +const MINIMUM_PUBLIC_SAMPLES = 3; + +function isPinnedProofReference(reference) { + if (!reference?.type || !/^[0-9a-f]{40}$/.test(reference?.commit || "")) return false; + try { + const url = new URL(reference.url); + if (url.protocol !== "https:") return false; + const escapedCommit = reference.commit.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + if (url.hostname === "github.com") { + return new RegExp(`/(?:commit|blob)/${escapedCommit}(?:/|$)`).test(url.pathname); + } + if (url.hostname === "raw.githubusercontent.com") { + return new RegExp(`^/[^/]+/[^/]+/${escapedCommit}/`).test(url.pathname); + } + return false; + } catch { + return false; + } +} + export function calculateDogfoodMetrics(register) { if (register.target_tasks !== 10 || !Array.isArray(register.tasks) || register.tasks.length !== 10) { throw new Error("dogfood register must contain exactly ten target tasks"); @@ -16,11 +36,23 @@ export function calculateDogfoodMetrics(register) { if (!Number.isFinite(task.time_to_proof_seconds) || task.time_to_proof_seconds <= 0 || !task.proof) { throw new Error(`${task.id}: completed tasks require positive time_to_proof_seconds and proof`); } + if (!Array.isArray(task.proof_references) || !task.proof_references.length) { + throw new Error(`${task.id}: completed tasks require proof_references`); + } + for (const reference of task.proof_references) { + if (!isPinnedProofReference(reference)) { + throw new Error(`${task.id}: proof_references require type and an HTTPS url pinned to the full commit`); + } + } } + const base = { + completed_tasks: completed.length, + target_tasks: 10, + minimum_public_samples: MINIMUM_PUBLIC_SAMPLES + }; if (!completed.length) { return { - completed_tasks: 0, - target_tasks: 10, + ...base, verified_completion_rate: null, sawyer_intervention_rate: null, median_time_to_proof_minutes: null, @@ -31,9 +63,17 @@ export function calculateDogfoodMetrics(register) { const middle = Math.floor(durations.length / 2); const medianSeconds = durations.length % 2 ? durations[middle] : (durations[middle - 1] + durations[middle]) / 2; const ratio = (count) => Number((count / completed.length).toFixed(4)); + if (completed.length < MINIMUM_PUBLIC_SAMPLES) { + return { + ...base, + verified_completion_rate: null, + sawyer_intervention_rate: null, + median_time_to_proof_minutes: null, + incorrect_stop_rate: null + }; + } return { - completed_tasks: completed.length, - target_tasks: 10, + ...base, verified_completion_rate: ratio(completed.filter((task) => task.verified).length), sawyer_intervention_rate: ratio(completed.filter((task) => task.sawyer_interventions > 0).length), median_time_to_proof_minutes: Number((medianSeconds / 60).toFixed(2)), @@ -46,6 +86,9 @@ export function renderDogfoodMarkdown(metrics) { const time = metrics.median_time_to_proof_minutes == null ? "Pending" : `${metrics.median_time_to_proof_minutes} min`; return `# Dogfood Report\n\n` + `Progress: **${metrics.completed_tasks}/${metrics.target_tasks} real tasks**\n\n` + + (metrics.completed_tasks < metrics.minimum_public_samples + ? `Performance rates remain **Pending** until ${metrics.minimum_public_samples} real tasks are recorded.\n\n` + : "") + `| Metric | Result |\n|---|---:|\n` + `| Verified completion rate | ${percent(metrics.verified_completion_rate)} |\n` + `| Sawyer intervention rate | ${percent(metrics.sawyer_intervention_rate)} |\n` + diff --git a/scripts/test-dogfood-metrics.mjs b/scripts/test-dogfood-metrics.mjs index 42660c4..20e6a6a 100644 --- a/scripts/test-dogfood-metrics.mjs +++ b/scripts/test-dogfood-metrics.mjs @@ -4,16 +4,45 @@ import { calculateDogfoodMetrics } from "./dogfood-metrics.mjs"; const pending = Array.from({ length: 10 }, (_, index) => ({ id: `DF-${String(index + 1).padStart(2, "0")}`, status: "pending" })); assert.equal(calculateDogfoodMetrics({ target_tasks: 10, tasks: pending }).verified_completion_rate, null); +const proofReferences = [{ + type: "commit", + url: "https://github.com/example/repo/commit/0123456789abcdef0123456789abcdef01234567", + commit: "0123456789abcdef0123456789abcdef01234567" +}]; const tasks = structuredClone(pending); -tasks[0] = { ...tasks[0], status: "completed", loopspine_invoked: true, verified: true, sawyer_interventions: 0, time_to_proof_seconds: 60, incorrect_stop: false, proof: "test A" }; -tasks[1] = { ...tasks[1], status: "completed", loopspine_invoked: true, verified: false, sawyer_interventions: 1, time_to_proof_seconds: 180, incorrect_stop: true, proof: "test B" }; +tasks[0] = { ...tasks[0], status: "completed", loopspine_invoked: true, verified: true, sawyer_interventions: 0, time_to_proof_seconds: 60, incorrect_stop: false, proof: "test A", proof_references: proofReferences }; +tasks[1] = { ...tasks[1], status: "completed", loopspine_invoked: true, verified: false, sawyer_interventions: 1, time_to_proof_seconds: 180, incorrect_stop: true, proof: "test B", proof_references: proofReferences }; assert.deepEqual(calculateDogfoodMetrics({ target_tasks: 10, tasks }), { completed_tasks: 2, target_tasks: 10, - verified_completion_rate: 0.5, - sawyer_intervention_rate: 0.5, - median_time_to_proof_minutes: 2, - incorrect_stop_rate: 0.5 + minimum_public_samples: 3, + verified_completion_rate: null, + sawyer_intervention_rate: null, + median_time_to_proof_minutes: null, + incorrect_stop_rate: null }); -console.log("Dogfood metric tests passed: empty and completed pilots."); +tasks[2] = { ...tasks[2], status: "completed", loopspine_invoked: true, verified: true, sawyer_interventions: 0, time_to_proof_seconds: 300, incorrect_stop: false, proof: "test C", proof_references: proofReferences }; +assert.deepEqual(calculateDogfoodMetrics({ target_tasks: 10, tasks }), { + completed_tasks: 3, + target_tasks: 10, + minimum_public_samples: 3, + verified_completion_rate: 0.6667, + sawyer_intervention_rate: 0.3333, + median_time_to_proof_minutes: 3, + incorrect_stop_rate: 0.3333 +}); + +const missingReferences = structuredClone(tasks); +delete missingReferences[0].proof_references; +assert.throws(() => calculateDogfoodMetrics({ target_tasks: 10, tasks: missingReferences }), /proof_references/); + +const mutableReference = structuredClone(tasks); +mutableReference[0].proof_references[0].url = "https://github.com/example/repo/blob/main/receipt.json"; +assert.throws(() => calculateDogfoodMetrics({ target_tasks: 10, tasks: mutableReference }), /pinned to the full commit/); + +const querySpoof = structuredClone(tasks); +querySpoof[0].proof_references[0].url = `https://github.com/example/repo/blob/main/receipt.json?commit=${proofReferences[0].commit}`; +assert.throws(() => calculateDogfoodMetrics({ target_tasks: 10, tasks: querySpoof }), /pinned to the full commit/); + +console.log("Dogfood metric tests passed: sample floor, completed pilots, and proof references.");