diff --git a/doc/EVIDENCE_GATE.md b/doc/EVIDENCE_GATE.md new file mode 100644 index 00000000000..4a1e1b5ab05 --- /dev/null +++ b/doc/EVIDENCE_GATE.md @@ -0,0 +1,15 @@ +# Evidence Gate Operations + +Transitions into `in_review` are rejected with HTTP 422 when required artifact evidence is missing. The response identifies the missing evidence shapes: + +```json +{"error":"missing-evidence","missing":["test-output","checklist:done-when"]} +``` + +An operator can authorize a time-limited exception by adding a user-authored issue comment in this exact form: + +```text +evidence-gate: override +``` + +The comment must be less than one hour old when the transition is attempted. Agent-authored comments, empty reasons, future timestamps, and expired comments are ignored. A successful override is recorded in `lastEvidenceVerdict` with `overridden: true` and `overrideReason`, and those fields are included in the server audit log. The gate only runs when entering `in_review`; updates to an issue already in review are grandfathered. diff --git a/packages/db/src/schema/issues.ts b/packages/db/src/schema/issues.ts index 7b3d6235311..347363a5cd2 100644 --- a/packages/db/src/schema/issues.ts +++ b/packages/db/src/schema/issues.ts @@ -95,6 +95,8 @@ export const issues = pgTable( missing: string[]; evidenceFound: string[]; unlabeledFallback: boolean; + overridden?: boolean; + overrideReason?: string; evaluatedAt: string; }>(), // Materialized from lastEvidenceVerdict.evaluatedAt when the evidence gate diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index f9feb85991a..9332dde00af 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -599,6 +599,8 @@ export interface Issue { missing: string[]; evidenceFound: string[]; unlabeledFallback: boolean; + overridden?: boolean; + overrideReason?: string; evaluatedAt: string; } | null; activeRecoveryAction?: IssueRecoveryAction | null; diff --git a/server/src/__tests__/error-handler.test.ts b/server/src/__tests__/error-handler.test.ts index 483403e047c..bcf4f3ab5a8 100644 --- a/server/src/__tests__/error-handler.test.ts +++ b/server/src/__tests__/error-handler.test.ts @@ -75,4 +75,22 @@ describe("errorHandler", () => { expect(res.err).toBe(err); expect(res.__errorContext?.error?.message).toBe("db exploded"); }); + + it("returns the evidence-gate 422 contract without a details wrapper", () => { + const req = makeReq(); + const res = makeRes(); + const next = vi.fn() as unknown as NextFunction; + const err = new HttpError(422, "missing-evidence", { + code: "missing-evidence", + missing: ["test-output", "checklist:done-when"], + }); + + errorHandler(err, req, res, next); + + expect(res.status).toHaveBeenCalledWith(422); + expect(res.json).toHaveBeenCalledWith({ + error: "missing-evidence", + missing: ["test-output", "checklist:done-when"], + }); + }); }); diff --git a/server/src/__tests__/evidence-gate-wiring.test.ts b/server/src/__tests__/evidence-gate-wiring.test.ts index a067d8175fd..e9c75054699 100644 --- a/server/src/__tests__/evidence-gate-wiring.test.ts +++ b/server/src/__tests__/evidence-gate-wiring.test.ts @@ -37,7 +37,7 @@ describe("runEvidenceGate", () => { ); const fixedNow = new Date("2026-05-11T22:00:00.000Z"); const result = await runEvidenceGate(fetch, "issue-1", fixedNow); - expect(fetch).toHaveBeenCalledWith("issue-1"); + expect(fetch).toHaveBeenCalledWith("issue-1", fixedNow); expect(result.verdict).toBe("pass"); expect(result.missing).toEqual([]); expect(result.unlabeledFallback).toBe(false); @@ -71,6 +71,85 @@ describe("runEvidenceGate", () => { ); }); + it("passes with the newest recent user-authored operator override", async () => { + const fetch = vi.fn<(id: string) => Promise>(async () => ({ + description: FRONTEND_DONE_WHEN, + labels: [{ name: "frontend" }], + comments: [ + { + body: "evidence-gate: override incident response requires landing now", + authorAgentId: null, + authorUserId: "operator-1", + createdAt: "2026-05-11T21:30:00.000Z", + }, + ], + workProducts: [], + })); + + const result = await runEvidenceGate(fetch, "issue-override", new Date("2026-05-11T22:00:00.000Z")); + + expect(result).toMatchObject({ + verdict: "pass", + overridden: true, + overrideReason: "incident response requires landing now", + missing: [], + }); + }); + + it("finds an operator override outside the evaluator comment window", async () => { + const override = { + body: "evidence-gate: override incident response requires landing now", + authorAgentId: null, + authorUserId: "operator-1", + createdAt: "2026-05-11T21:30:00.000Z", + }; + const fetch = vi.fn<(id: string) => Promise>(async () => ({ + description: FRONTEND_DONE_WHEN, + labels: [{ name: "frontend" }], + comments: Array.from({ length: 10 }, (_, index) => ({ + body: `later agent comment ${index}`, + authorAgentId: "agent-1", + authorUserId: null, + createdAt: `2026-05-11T21:4${index}:00.000Z`, + })), + operatorOverrideComments: [override], + workProducts: [], + })); + + const result = await runEvidenceGate(fetch, "issue-displaced-override", new Date("2026-05-11T22:00:00.000Z")); + + expect(result).toMatchObject({ + verdict: "pass", + overridden: true, + overrideReason: "incident response requires landing now", + }); + }); + + it.each([ + ["agent-authored", "agent-1", null, "2026-05-11T21:30:00.000Z"], + ["expired", null, "operator-1", "2026-05-11T20:59:59.999Z"], + ["future", null, "operator-1", "2026-05-11T22:00:00.001Z"], + ["malformed", null, "operator-1", "2026-05-11T21:30:00.000Z"], + ["blank", null, "operator-1", "2026-05-11T21:30:00.000Z"], + ])("ignores %s override comments", async (kind, authorAgentId, authorUserId, createdAt) => { + const body = kind === "malformed" + ? "evidence-gate: override" + : kind === "blank" + ? "evidence-gate: override " + : "evidence-gate: override reason"; + const fetch = vi.fn<(id: string) => Promise>(async () => ({ + description: FRONTEND_DONE_WHEN, + labels: [{ name: "frontend" }], + comments: [{ body, authorAgentId, authorUserId, createdAt }], + workProducts: [], + })); + + const result = await runEvidenceGate(fetch, `issue-${kind}`, new Date("2026-05-11T22:00:00.000Z")); + + expect(result.verdict).toBe("block"); + expect(result.overridden).toBeUndefined(); + }); + it("maps work-product `type` to evaluator `kind` (screenshot pickup)", async () => { const fetch = vi.fn<(id: string) => Promise>( async () => ({ diff --git a/server/src/__tests__/issues-patch-evidence.test.ts b/server/src/__tests__/issues-patch-evidence.test.ts new file mode 100644 index 00000000000..79b3514295b --- /dev/null +++ b/server/src/__tests__/issues-patch-evidence.test.ts @@ -0,0 +1,158 @@ +import { randomUUID } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + activityLog, + companies, + createDb, + issueComments, + issueLabels, + issues, + labels, +} from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { errorHandler } from "../middleware/index.js"; +import { issueRoutes } from "../routes/issues.js"; + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres issue evidence PATCH tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describeEmbeddedPostgres("PATCH /issues/:id evidence gate", () => { + let tempDb: Awaited> | null = null; + let db: ReturnType; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-patch-evidence-"); + db = createDb(tempDb.connectionString); + }, 30_000); + + afterEach(async () => { + await db.delete(issueLabels); + await db.delete(issueComments); + await db.delete(activityLog); + await db.delete(issues); + await db.delete(labels); + await db.delete(companies); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + function createApp() { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { + (req as any).actor = { type: "board", source: "local_implicit" }; + next(); + }); + app.use("/api", issueRoutes(db, {} as any)); + app.use(errorHandler); + return app; + } + + async function seedFrontendIssue(status: "in_progress" | "in_review" = "in_progress") { + const companyId = randomUUID(); + const issueId = randomUUID(); + const labelId = randomUUID(); + const prefix = `EG${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}`; + + await db.insert(companies).values({ + id: companyId, + name: "Evidence Gate Co", + issuePrefix: prefix, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(labels).values({ + id: labelId, + companyId, + name: "frontend", + color: "#000000", + }); + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Ship frontend change", + description: "## Done when\n- desktop works\n- mobile works\n- tests pass", + status, + priority: "medium", + issueNumber: 1, + identifier: `${prefix}-1`, + }); + await db.insert(issueLabels).values({ issueId, labelId, companyId }); + + return { companyId, issueId }; + } + + it("rejects a computed block with the structured 422 contract and rolls back", async () => { + const { issueId } = await seedFrontendIssue(); + + const response = await request(createApp()) + .patch(`/api/issues/${issueId}`) + .send({ status: "in_review" }); + + expect(response.status).toBe(422); + expect(response.body).toEqual({ + error: "missing-evidence", + missing: expect.arrayContaining([ + "screenshot:1440x900", + "screenshot:390x844", + "checklist:done-when", + ]), + }); + const [persisted] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(persisted).toMatchObject({ status: "in_progress", lastEvidenceVerdict: null }); + }); + + it("accepts a recent user override and persists its audit fields", async () => { + const { companyId, issueId } = await seedFrontendIssue(); + await db.insert(issueComments).values({ + companyId, + issueId, + body: "evidence-gate: override incident response requires landing now", + authorUserId: "operator-1", + authorAgentId: null, + createdAt: new Date(), + }); + + const response = await request(createApp()) + .patch(`/api/issues/${issueId}`) + .send({ status: "in_review" }); + + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body).toMatchObject({ + status: "in_review", + lastEvidenceVerdict: { + verdict: "pass", + overridden: true, + overrideReason: "incident response requires landing now", + }, + }); + }); + + it("grandfathers updates to an issue already in review", async () => { + const { issueId } = await seedFrontendIssue("in_review"); + + const response = await request(createApp()) + .patch(`/api/issues/${issueId}`) + .send({ title: "Updated while under review" }); + + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body).toMatchObject({ + status: "in_review", + title: "Updated while under review", + lastEvidenceVerdict: null, + }); + }); +}); diff --git a/server/src/middleware/error-handler.ts b/server/src/middleware/error-handler.ts index c455cbad950..cfb103ecfc3 100644 --- a/server/src/middleware/error-handler.ts +++ b/server/src/middleware/error-handler.ts @@ -50,10 +50,20 @@ export function errorHandler( const tc = getTelemetryClient(); if (tc) trackErrorHandlerCrash(tc, { errorCode: err.name }); } - res.status(err.status).json({ - error: err.message, - ...(err.details ? { details: err.details } : {}), - }); + if ( + err.status === 422 && + err.message === "missing-evidence" && + typeof err.details === "object" && + err.details !== null && + Array.isArray((err.details as { missing?: unknown }).missing) + ) { + res.status(422).json({ + error: "missing-evidence", + missing: (err.details as { missing: unknown[] }).missing, + }); + return; + } + res.status(err.status).json({ error: err.message, ...(err.details ? { details: err.details } : {}) }); return; } diff --git a/server/src/services/evidence-gate-wiring.ts b/server/src/services/evidence-gate-wiring.ts index 3265c5d553c..93a606360b9 100644 --- a/server/src/services/evidence-gate-wiring.ts +++ b/server/src/services/evidence-gate-wiring.ts @@ -21,6 +21,7 @@ export interface EvidenceFetchResult { doneWhenBulletsRemoved?: boolean; labels: Array<{ name: string }>; comments: EvidenceCommentLite[]; + operatorOverrideComments?: EvidenceCommentLite[]; workProducts: Array<{ type: string; metadata: Record | null; @@ -30,6 +31,7 @@ export interface EvidenceFetchResult { export type FetchEvidenceForGate = ( issueId: string, + now: Date, ) => Promise; export interface EvidenceVerdictRecord { @@ -40,9 +42,14 @@ export interface EvidenceVerdictRecord { allDetected: string[]; unlabeledFallback: boolean; diagnostics: string[]; + overridden?: boolean; + overrideReason?: string; evaluatedAt: string; } +const OPERATOR_OVERRIDE_PATTERN = /^evidence-gate: override (.+)$/; +const OPERATOR_OVERRIDE_MAX_AGE_MS = 60 * 60 * 1000; + /** * Run the gate for one issue. Returns the verdict record the caller should * persist to `issues.lastEvidenceVerdict`. Caller is responsible for @@ -63,7 +70,35 @@ export async function runEvidenceGate( issueId: string, now: Date = new Date(), ): Promise { - const data = await fetch(issueId); + const data = await fetch(issueId, now); + const override = (data.operatorOverrideComments ?? data.comments) + .filter((comment) => comment.authorUserId !== null && comment.authorAgentId === null) + .map((comment) => ({ + createdAt: new Date(comment.createdAt).getTime(), + match: OPERATOR_OVERRIDE_PATTERN.exec(comment.body), + })) + .filter(({ createdAt, match }) => + match !== null && + match[1]!.trim().length > 0 && + Number.isFinite(createdAt) && + createdAt <= now.getTime() && + now.getTime() - createdAt <= OPERATOR_OVERRIDE_MAX_AGE_MS + ) + .sort((a, b) => b.createdAt - a.createdAt)[0]; + if (override?.match) { + return { + verdict: "pass", + missing: [], + evidenceFound: [], + requiredFound: [], + allDetected: [], + unlabeledFallback: false, + diagnostics: [], + overridden: true, + overrideReason: override.match[1]!.trim(), + evaluatedAt: now.toISOString(), + }; + } const evaluation = evaluateEvidence({ issue: { description: data.description, diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index c442eabedf7..ae7f38e43b6 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,6 +1,6 @@ import { Buffer } from "node:buffer"; import { createHash } from "node:crypto"; -import { and, asc, desc, eq, exists, gt, inArray, isNotNull, isNull, like, lt, ne, notInArray, or, sql, type SQL } from "drizzle-orm"; +import { and, asc, desc, eq, exists, gt, gte, inArray, isNotNull, isNull, like, lt, lte, ne, notInArray, or, sql, type SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { activityLog, @@ -97,7 +97,6 @@ import { import { runEvidenceGate, type EvidenceFetchResult } from "./evidence-gate-wiring.js"; import { countDoneWhenBullets } from "./evidence-gate.js"; import { shouldBlockNarratedDone } from "./done-gate.js"; -import { shouldBlockUnreviewableInReview } from "./in-review-gate.js"; import { parseIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS, @@ -1472,17 +1471,18 @@ async function withIssueLabels(dbOrTx: any, rows: IssueRow[]): Promise { - const [recentComments, workProductRows, labelsByIssueId, descriptionHistory] = await Promise.all([ + const [recentComments, operatorOverrideComments, workProductRows, labelsByIssueId, descriptionHistory] = await Promise.all([ dbOrTx .select({ body: issueComments.body, @@ -1494,6 +1494,23 @@ async function fetchEvidenceForIssue( .where(eq(issueComments.issueId, issueId)) .orderBy(desc(issueComments.createdAt)) .limit(10), + dbOrTx + .select({ + body: issueComments.body, + authorAgentId: issueComments.authorAgentId, + authorUserId: issueComments.authorUserId, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where(and( + eq(issueComments.issueId, issueId), + isNotNull(issueComments.authorUserId), + isNull(issueComments.authorAgentId), + like(issueComments.body, "evidence-gate: override %"), + gte(issueComments.createdAt, new Date(now.getTime() - 60 * 60 * 1000)), + lte(issueComments.createdAt, now), + )) + .orderBy(desc(issueComments.createdAt)), dbOrTx .select({ type: issueWorkProducts.type, @@ -1527,6 +1544,7 @@ async function fetchEvidenceForIssue( countDoneWhenBullets(description ?? "") === 0 && hadPriorDoneWhenBullets, labels: issueLabels.map((l: { name: string }) => ({ name: l.name })), comments: recentComments as EvidenceFetchResult["comments"], + operatorOverrideComments: operatorOverrideComments as EvidenceFetchResult["operatorOverrideComments"], workProducts: workProductRows as EvidenceFetchResult["workProducts"], }; } @@ -6504,11 +6522,12 @@ export function issueService(db: Db) { if (experimental.enableDoneExecutionGate && shouldBlockNarratedDone(doneGateInput)) { try { doneTransitionEvidenceVerdict = await runEvidenceGate( - (issueId) => fetchEvidenceForIssue( + (issueId, now) => fetchEvidenceForIssue( dbOrTx, issueId, issueData.description !== undefined ? issueData.description : existing.description, existing.description, + now, ), id, ); @@ -6627,16 +6646,8 @@ export function issueService(db: Db) { applyStatusSideEffects(issueData.status, patch); - // Phase-1 warn-only evidence gate (BLO-4824 / BLO-4461). Records the - // gate's verdict but never throws on its own — Phase 2 (BLO-4828) will - // flip `block` to a 422. Errors during evaluation are swallowed so a - // misbehaving gate can't break the PATCH; production runs surface - // them via the warn log. - // - // The flag-gated in_review enforcement below intentionally lives - // OUTSIDE the try/catch: an evaluation failure yields verdict == null - // (gate fails open), while a successfully-computed non-pass verdict - // throws without being swallowed. + // Evaluation failures remain fail-open, but a computed block verdict + // rejects every new transition to in_review. if ( issueData.status === "in_review" && existing.status !== "in_review" @@ -6644,11 +6655,12 @@ export function issueService(db: Db) { let inReviewVerdict: Awaited> | null = null; try { const verdict = await runEvidenceGate( - (issueId) => fetchEvidenceForIssue( + (issueId, now) => fetchEvidenceForIssue( dbOrTx, issueId, issueData.description !== undefined ? issueData.description : existing.description, existing.description, + now, ), id, ); @@ -6664,6 +6676,8 @@ export function issueService(db: Db) { evidenceFound: verdict.evidenceFound, unlabeledFallback: verdict.unlabeledFallback, diagnostics: verdict.diagnostics, + overridden: verdict.overridden, + overrideReason: verdict.overrideReason, }, `evidence-gate: ${verdict.verdict} on in_review transition`, ); @@ -6671,38 +6685,19 @@ export function issueService(db: Db) { logger.warn( { issueId: id, - err: err instanceof Error ? err.message : String(err), + err, }, "evidence-gate: evaluation failed; proceeding without verdict", ); } - // In-review evidence gate (narrated-review hardening, instance flag - // `enableInReviewEvidenceGate`, default off). Blocks an agent - // flipping an issue to `in_review` when the evidence gate found - // nothing reviewable — the failure mode where an agent does - // analysis-only work and narrates in_review with no PR, branch, or - // commits. Never gates human actors. See - // server/src/services/in-review-gate.ts. - if ( - experimental.enableInReviewEvidenceGate && - shouldBlockUnreviewableInReview({ - fromStatus: existing.status, - toStatus: issueData.status, - isAgentActor: actorAgentId != null, - verdict: inReviewVerdict, - }) - ) { - throw unprocessable( - "Issue cannot be moved to in_review without reviewable evidence " + - `(missing: ${inReviewVerdict?.missing.join(", ") || "unknown"})`, - { - reason: "in_review_without_reviewable_evidence", - issueId: id, - missing: inReviewVerdict?.missing ?? [], - }, - ); + if (inReviewVerdict?.verdict === "block") { + throw unprocessable("missing-evidence", { + code: "missing-evidence", + missing: inReviewVerdict.missing, + }); } + } if (issueData.status && issueData.status !== "done") {