From 45b017a5e906b48f3897a7e3bc050c16934fc593 Mon Sep 17 00:00:00 2001 From: kkroo Date: Sun, 12 Jul 2026 16:48:39 +0000 Subject: [PATCH 1/3] feat(evidence-gate): enforce review evidence --- doc/EVIDENCE_GATE.md | 15 ++++++ packages/db/src/schema/issues.ts | 2 + packages/shared/src/types/issue.ts | 2 + server/src/__tests__/error-handler.test.ts | 18 +++++++ .../__tests__/evidence-gate-wiring.test.ts | 50 +++++++++++++++++++ server/src/middleware/error-handler.ts | 18 +++++-- server/src/services/evidence-gate-wiring.ts | 33 ++++++++++++ server/src/services/issues.ts | 21 ++++---- 8 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 doc/EVIDENCE_GATE.md diff --git a/doc/EVIDENCE_GATE.md b/doc/EVIDENCE_GATE.md new file mode 100644 index 00000000000..4a1e1b5ab05 --- /dev/null +++ b/doc/EVIDENCE_GATE.md @@ -0,0 +1,15 @@ +# Evidence Gate Operations + +Transitions into `in_review` are rejected with HTTP 422 when required artifact evidence is missing. The response identifies the missing evidence shapes: + +```json +{"error":"missing-evidence","missing":["test-output","checklist:done-when"]} +``` + +An operator can authorize a time-limited exception by adding a user-authored issue comment in this exact form: + +```text +evidence-gate: override +``` + +The comment must be less than one hour old when the transition is attempted. Agent-authored comments, empty reasons, future timestamps, and expired comments are ignored. A successful override is recorded in `lastEvidenceVerdict` with `overridden: true` and `overrideReason`, and those fields are included in the server audit log. The gate only runs when entering `in_review`; updates to an issue already in review are grandfathered. diff --git a/packages/db/src/schema/issues.ts b/packages/db/src/schema/issues.ts index 7b3d6235311..347363a5cd2 100644 --- a/packages/db/src/schema/issues.ts +++ b/packages/db/src/schema/issues.ts @@ -95,6 +95,8 @@ export const issues = pgTable( missing: string[]; evidenceFound: string[]; unlabeledFallback: boolean; + overridden?: boolean; + overrideReason?: string; evaluatedAt: string; }>(), // Materialized from lastEvidenceVerdict.evaluatedAt when the evidence gate diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index f9feb85991a..9332dde00af 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -599,6 +599,8 @@ export interface Issue { missing: string[]; evidenceFound: string[]; unlabeledFallback: boolean; + overridden?: boolean; + overrideReason?: string; evaluatedAt: string; } | null; activeRecoveryAction?: IssueRecoveryAction | null; diff --git a/server/src/__tests__/error-handler.test.ts b/server/src/__tests__/error-handler.test.ts index 483403e047c..bcf4f3ab5a8 100644 --- a/server/src/__tests__/error-handler.test.ts +++ b/server/src/__tests__/error-handler.test.ts @@ -75,4 +75,22 @@ describe("errorHandler", () => { expect(res.err).toBe(err); expect(res.__errorContext?.error?.message).toBe("db exploded"); }); + + it("returns the evidence-gate 422 contract without a details wrapper", () => { + const req = makeReq(); + const res = makeRes(); + const next = vi.fn() as unknown as NextFunction; + const err = new HttpError(422, "missing-evidence", { + code: "missing-evidence", + missing: ["test-output", "checklist:done-when"], + }); + + errorHandler(err, req, res, next); + + expect(res.status).toHaveBeenCalledWith(422); + expect(res.json).toHaveBeenCalledWith({ + error: "missing-evidence", + missing: ["test-output", "checklist:done-when"], + }); + }); }); diff --git a/server/src/__tests__/evidence-gate-wiring.test.ts b/server/src/__tests__/evidence-gate-wiring.test.ts index a067d8175fd..e6a86498746 100644 --- a/server/src/__tests__/evidence-gate-wiring.test.ts +++ b/server/src/__tests__/evidence-gate-wiring.test.ts @@ -71,6 +71,56 @@ describe("runEvidenceGate", () => { ); }); + it("passes with the newest recent user-authored operator override", async () => { + const fetch = vi.fn<(id: string) => Promise>(async () => ({ + description: FRONTEND_DONE_WHEN, + labels: [{ name: "frontend" }], + comments: [ + { + body: "evidence-gate: override incident response requires landing now", + authorAgentId: null, + authorUserId: "operator-1", + createdAt: "2026-05-11T21:30:00.000Z", + }, + ], + workProducts: [], + })); + + const result = await runEvidenceGate(fetch, "issue-override", new Date("2026-05-11T22:00:00.000Z")); + + expect(result).toMatchObject({ + verdict: "pass", + overridden: true, + overrideReason: "incident response requires landing now", + missing: [], + }); + }); + + it.each([ + ["agent-authored", "agent-1", null, "2026-05-11T21:30:00.000Z"], + ["expired", null, "operator-1", "2026-05-11T20:59:59.999Z"], + ["future", null, "operator-1", "2026-05-11T22:00:00.001Z"], + ["malformed", null, "operator-1", "2026-05-11T21:30:00.000Z"], + ["blank", null, "operator-1", "2026-05-11T21:30:00.000Z"], + ])("ignores %s override comments", async (kind, authorAgentId, authorUserId, createdAt) => { + const body = kind === "malformed" + ? "evidence-gate: override" + : kind === "blank" + ? "evidence-gate: override " + : "evidence-gate: override reason"; + const fetch = vi.fn<(id: string) => Promise>(async () => ({ + description: FRONTEND_DONE_WHEN, + labels: [{ name: "frontend" }], + comments: [{ body, authorAgentId, authorUserId, createdAt }], + workProducts: [], + })); + + const result = await runEvidenceGate(fetch, `issue-${kind}`, new Date("2026-05-11T22:00:00.000Z")); + + expect(result.verdict).toBe("block"); + expect(result.overridden).toBeUndefined(); + }); + it("maps work-product `type` to evaluator `kind` (screenshot pickup)", async () => { const fetch = vi.fn<(id: string) => Promise>( async () => ({ diff --git a/server/src/middleware/error-handler.ts b/server/src/middleware/error-handler.ts index c455cbad950..cfb103ecfc3 100644 --- a/server/src/middleware/error-handler.ts +++ b/server/src/middleware/error-handler.ts @@ -50,10 +50,20 @@ export function errorHandler( const tc = getTelemetryClient(); if (tc) trackErrorHandlerCrash(tc, { errorCode: err.name }); } - res.status(err.status).json({ - error: err.message, - ...(err.details ? { details: err.details } : {}), - }); + if ( + err.status === 422 && + err.message === "missing-evidence" && + typeof err.details === "object" && + err.details !== null && + Array.isArray((err.details as { missing?: unknown }).missing) + ) { + res.status(422).json({ + error: "missing-evidence", + missing: (err.details as { missing: unknown[] }).missing, + }); + return; + } + res.status(err.status).json({ error: err.message, ...(err.details ? { details: err.details } : {}) }); return; } diff --git a/server/src/services/evidence-gate-wiring.ts b/server/src/services/evidence-gate-wiring.ts index 3265c5d553c..2a2fbe217e2 100644 --- a/server/src/services/evidence-gate-wiring.ts +++ b/server/src/services/evidence-gate-wiring.ts @@ -40,9 +40,14 @@ export interface EvidenceVerdictRecord { allDetected: string[]; unlabeledFallback: boolean; diagnostics: string[]; + overridden?: boolean; + overrideReason?: string; evaluatedAt: string; } +const OPERATOR_OVERRIDE_PATTERN = /^evidence-gate: override (.+)$/; +const OPERATOR_OVERRIDE_MAX_AGE_MS = 60 * 60 * 1000; + /** * Run the gate for one issue. Returns the verdict record the caller should * persist to `issues.lastEvidenceVerdict`. Caller is responsible for @@ -64,6 +69,34 @@ export async function runEvidenceGate( now: Date = new Date(), ): Promise { const data = await fetch(issueId); + const override = data.comments + .filter((comment) => comment.authorUserId !== null && comment.authorAgentId === null) + .map((comment) => ({ + createdAt: new Date(comment.createdAt).getTime(), + match: OPERATOR_OVERRIDE_PATTERN.exec(comment.body), + })) + .filter(({ createdAt, match }) => + match !== null && + match[1]!.trim().length > 0 && + Number.isFinite(createdAt) && + createdAt <= now.getTime() && + now.getTime() - createdAt <= OPERATOR_OVERRIDE_MAX_AGE_MS + ) + .sort((a, b) => b.createdAt - a.createdAt)[0]; + if (override?.match) { + return { + verdict: "pass", + missing: [], + evidenceFound: [], + requiredFound: [], + allDetected: [], + unlabeledFallback: false, + diagnostics: [], + overridden: true, + overrideReason: override.match[1]!.trim(), + evaluatedAt: now.toISOString(), + }; + } const evaluation = evaluateEvidence({ issue: { description: data.description, diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index c442eabedf7..adc2c3d2e14 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -6627,16 +6627,8 @@ export function issueService(db: Db) { applyStatusSideEffects(issueData.status, patch); - // Phase-1 warn-only evidence gate (BLO-4824 / BLO-4461). Records the - // gate's verdict but never throws on its own — Phase 2 (BLO-4828) will - // flip `block` to a 422. Errors during evaluation are swallowed so a - // misbehaving gate can't break the PATCH; production runs surface - // them via the warn log. - // - // The flag-gated in_review enforcement below intentionally lives - // OUTSIDE the try/catch: an evaluation failure yields verdict == null - // (gate fails open), while a successfully-computed non-pass verdict - // throws without being swallowed. + // Evaluation failures remain fail-open, but a computed block verdict + // rejects every new transition to in_review. if ( issueData.status === "in_review" && existing.status !== "in_review" @@ -6664,6 +6656,8 @@ export function issueService(db: Db) { evidenceFound: verdict.evidenceFound, unlabeledFallback: verdict.unlabeledFallback, diagnostics: verdict.diagnostics, + overridden: verdict.overridden, + overrideReason: verdict.overrideReason, }, `evidence-gate: ${verdict.verdict} on in_review transition`, ); @@ -6677,6 +6671,13 @@ export function issueService(db: Db) { ); } + if (inReviewVerdict?.verdict === "block") { + throw unprocessable("missing-evidence", { + code: "missing-evidence", + missing: inReviewVerdict.missing, + }); + } + // In-review evidence gate (narrated-review hardening, instance flag // `enableInReviewEvidenceGate`, default off). Blocks an agent // flipping an issue to `in_review` when the evidence gate found From 7c8182003672fbb57612c8378dd148e8408305a8 Mon Sep 17 00:00:00 2001 From: Search Date: Sun, 12 Jul 2026 17:02:04 +0000 Subject: [PATCH 2/3] fix(evidence-gate): preserve displaced overrides --- .../__tests__/evidence-gate-wiring.test.ts | 31 ++++++++++++++++- server/src/services/evidence-gate-wiring.ts | 6 ++-- server/src/services/issues.ts | 33 +++++++++++++++---- 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/server/src/__tests__/evidence-gate-wiring.test.ts b/server/src/__tests__/evidence-gate-wiring.test.ts index e6a86498746..e9c75054699 100644 --- a/server/src/__tests__/evidence-gate-wiring.test.ts +++ b/server/src/__tests__/evidence-gate-wiring.test.ts @@ -37,7 +37,7 @@ describe("runEvidenceGate", () => { ); const fixedNow = new Date("2026-05-11T22:00:00.000Z"); const result = await runEvidenceGate(fetch, "issue-1", fixedNow); - expect(fetch).toHaveBeenCalledWith("issue-1"); + expect(fetch).toHaveBeenCalledWith("issue-1", fixedNow); expect(result.verdict).toBe("pass"); expect(result.missing).toEqual([]); expect(result.unlabeledFallback).toBe(false); @@ -96,6 +96,35 @@ describe("runEvidenceGate", () => { }); }); + it("finds an operator override outside the evaluator comment window", async () => { + const override = { + body: "evidence-gate: override incident response requires landing now", + authorAgentId: null, + authorUserId: "operator-1", + createdAt: "2026-05-11T21:30:00.000Z", + }; + const fetch = vi.fn<(id: string) => Promise>(async () => ({ + description: FRONTEND_DONE_WHEN, + labels: [{ name: "frontend" }], + comments: Array.from({ length: 10 }, (_, index) => ({ + body: `later agent comment ${index}`, + authorAgentId: "agent-1", + authorUserId: null, + createdAt: `2026-05-11T21:4${index}:00.000Z`, + })), + operatorOverrideComments: [override], + workProducts: [], + })); + + const result = await runEvidenceGate(fetch, "issue-displaced-override", new Date("2026-05-11T22:00:00.000Z")); + + expect(result).toMatchObject({ + verdict: "pass", + overridden: true, + overrideReason: "incident response requires landing now", + }); + }); + it.each([ ["agent-authored", "agent-1", null, "2026-05-11T21:30:00.000Z"], ["expired", null, "operator-1", "2026-05-11T20:59:59.999Z"], diff --git a/server/src/services/evidence-gate-wiring.ts b/server/src/services/evidence-gate-wiring.ts index 2a2fbe217e2..93a606360b9 100644 --- a/server/src/services/evidence-gate-wiring.ts +++ b/server/src/services/evidence-gate-wiring.ts @@ -21,6 +21,7 @@ export interface EvidenceFetchResult { doneWhenBulletsRemoved?: boolean; labels: Array<{ name: string }>; comments: EvidenceCommentLite[]; + operatorOverrideComments?: EvidenceCommentLite[]; workProducts: Array<{ type: string; metadata: Record | null; @@ -30,6 +31,7 @@ export interface EvidenceFetchResult { export type FetchEvidenceForGate = ( issueId: string, + now: Date, ) => Promise; export interface EvidenceVerdictRecord { @@ -68,8 +70,8 @@ export async function runEvidenceGate( issueId: string, now: Date = new Date(), ): Promise { - const data = await fetch(issueId); - const override = data.comments + const data = await fetch(issueId, now); + const override = (data.operatorOverrideComments ?? data.comments) .filter((comment) => comment.authorUserId !== null && comment.authorAgentId === null) .map((comment) => ({ createdAt: new Date(comment.createdAt).getTime(), diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index adc2c3d2e14..b8a720b5bfe 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1472,17 +1472,18 @@ async function withIssueLabels(dbOrTx: any, rows: IssueRow[]): Promise { - const [recentComments, workProductRows, labelsByIssueId, descriptionHistory] = await Promise.all([ + const [recentComments, operatorOverrideComments, workProductRows, labelsByIssueId, descriptionHistory] = await Promise.all([ dbOrTx .select({ body: issueComments.body, @@ -1494,6 +1495,23 @@ async function fetchEvidenceForIssue( .where(eq(issueComments.issueId, issueId)) .orderBy(desc(issueComments.createdAt)) .limit(10), + dbOrTx + .select({ + body: issueComments.body, + authorAgentId: issueComments.authorAgentId, + authorUserId: issueComments.authorUserId, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where(and( + eq(issueComments.issueId, issueId), + isNotNull(issueComments.authorUserId), + isNull(issueComments.authorAgentId), + like(issueComments.body, "evidence-gate: override %"), + sql`${issueComments.createdAt} >= ${new Date(now.getTime() - 60 * 60 * 1000)}`, + sql`${issueComments.createdAt} <= ${now}`, + )) + .orderBy(desc(issueComments.createdAt)), dbOrTx .select({ type: issueWorkProducts.type, @@ -1527,6 +1545,7 @@ async function fetchEvidenceForIssue( countDoneWhenBullets(description ?? "") === 0 && hadPriorDoneWhenBullets, labels: issueLabels.map((l: { name: string }) => ({ name: l.name })), comments: recentComments as EvidenceFetchResult["comments"], + operatorOverrideComments: operatorOverrideComments as EvidenceFetchResult["operatorOverrideComments"], workProducts: workProductRows as EvidenceFetchResult["workProducts"], }; } @@ -6504,11 +6523,12 @@ export function issueService(db: Db) { if (experimental.enableDoneExecutionGate && shouldBlockNarratedDone(doneGateInput)) { try { doneTransitionEvidenceVerdict = await runEvidenceGate( - (issueId) => fetchEvidenceForIssue( + (issueId, now) => fetchEvidenceForIssue( dbOrTx, issueId, issueData.description !== undefined ? issueData.description : existing.description, existing.description, + now, ), id, ); @@ -6636,11 +6656,12 @@ export function issueService(db: Db) { let inReviewVerdict: Awaited> | null = null; try { const verdict = await runEvidenceGate( - (issueId) => fetchEvidenceForIssue( + (issueId, now) => fetchEvidenceForIssue( dbOrTx, issueId, issueData.description !== undefined ? issueData.description : existing.description, existing.description, + now, ), id, ); From 65f1ed9939862ac32770465302b5db4bd3440850 Mon Sep 17 00:00:00 2001 From: kkroo Date: Sun, 12 Jul 2026 17:13:39 +0000 Subject: [PATCH 3/3] fix(evidence-gate): remove superseded review guard --- server/src/services/issues.ts | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index b8a720b5bfe..e25285f6167 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -97,7 +97,6 @@ import { import { runEvidenceGate, type EvidenceFetchResult } from "./evidence-gate-wiring.js"; import { countDoneWhenBullets } from "./evidence-gate.js"; import { shouldBlockNarratedDone } from "./done-gate.js"; -import { shouldBlockUnreviewableInReview } from "./in-review-gate.js"; import { parseIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS, @@ -6699,32 +6698,6 @@ export function issueService(db: Db) { }); } - // In-review evidence gate (narrated-review hardening, instance flag - // `enableInReviewEvidenceGate`, default off). Blocks an agent - // flipping an issue to `in_review` when the evidence gate found - // nothing reviewable — the failure mode where an agent does - // analysis-only work and narrates in_review with no PR, branch, or - // commits. Never gates human actors. See - // server/src/services/in-review-gate.ts. - if ( - experimental.enableInReviewEvidenceGate && - shouldBlockUnreviewableInReview({ - fromStatus: existing.status, - toStatus: issueData.status, - isAgentActor: actorAgentId != null, - verdict: inReviewVerdict, - }) - ) { - throw unprocessable( - "Issue cannot be moved to in_review without reviewable evidence " + - `(missing: ${inReviewVerdict?.missing.join(", ") || "unknown"})`, - { - reason: "in_review_without_reviewable_evidence", - issueId: id, - missing: inReviewVerdict?.missing ?? [], - }, - ); - } } if (issueData.status && issueData.status !== "done") {