From 4c8e59eb727b87be74f1dc4447662ba612a4fd0e Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Sun, 25 Oct 2026 22:13:07 +0800 Subject: [PATCH 1/3] feat: derive offline goal checkpoints and next actions Reuse read-only comparison evidence, bind explicit owner decisions to pinned criteria, and render deterministic continuation views. Agent-assisted-by: Codex (implementation) Signed-off-by: GUOGUO --- docs/goal-checkpoints.md | 181 +++++++++++++++++++ docs/operations.md | 1 + tests/test_goal_checkpoint.py | 290 +++++++++++++++++++++++++++++++ tools/goal_checkpoint.py | 319 ++++++++++++++++++++++++++++++++++ tools/skill_eval_compare.py | 7 +- 5 files changed, 797 insertions(+), 1 deletion(-) create mode 100644 docs/goal-checkpoints.md create mode 100644 tests/test_goal_checkpoint.py create mode 100644 tools/goal_checkpoint.py diff --git a/docs/goal-checkpoints.md b/docs/goal-checkpoints.md new file mode 100644 index 00000000..4da7fac5 --- /dev/null +++ b/docs/goal-checkpoints.md @@ -0,0 +1,181 @@ +# Offline goal checkpoints + +`python -m tools.goal_checkpoint` rebuilds a goal view from explicit owner +records and one existing [configured skill comparison](skill-eval-comparison.md). +It reads the retained artifacts through the comparison controller's +`read_report` function. It does not call a model or write comparison reports. + +```mermaid +flowchart LR + G["Explicit scope and criteria"] --> C["Derived checkpoint"] + E["Existing plan, receipts and usage"] --> C + O["Pinned owner decisions"] --> C + C --> V["Markdown or JSON view"] + V --> A["Agent proposes the next step"] +``` + +The goal input is a small, explicit description of accepted scope and observed +decisions, not a runner configuration. The view is disposable: rebuild it from +the same pinned inputs instead of updating a second status history. It cannot +resume or retry an experiment, authorize spending, close an Issue, change a +verdict or promote knowledge. There is no scheduler or persistent registry. + +## Inputs and ownership + +Run from a development checkout, using its existing dependencies. The selected +comparison must have been prepared by that controller version. Its frozen +controller/file checks and retained result receipts remain authoritative. +Historical comparisons need their original controller checkout; a checkpoint +does not rewrite their freeze to make them readable by a newer controller. + +The caller supplies one UTF-8 JSON goal file: + +```json +{ + "schema_version": 1, + "id": "", + "title": "", + "scope": {"path": "scope.md", "sha256": ""}, + "acceptance_owner": "", + "criteria": [ + { + "id": "", + "description": "", + "evidence": [], + "acceptance": null + } + ], + "dependencies": [], + "next_step": "", + "plan_sha256": "", + "authorization": null +} +``` + +Replace placeholders with actual values. Unknown `acceptance_owner` is JSON +`null`, not the string `"null"`. Criteria must be nonempty with unique IDs. +Every file reference uses a path relative to the goal file and the SHA-256 of +its actual bytes. Evidence files may be outside that directory through an +explicit relative path. Only name files the caller is authorized to read; no +private sessions, credentials or automatic Issue scraping are inputs. + +| Input | Meaning | +| --- | --- | +| `scope` | Pinned accepted scope/decision supplied by the authorized caller; its prose is never parsed for authority | +| Criterion `evidence` | Explicit supporting files, each with `path` and `sha256`; an empty list remains incomplete | +| Criterion `acceptance` | Optional pinned JSON decision from the named acceptance owner, bound to this exact criterion and evidence | +| `dependencies` | Explicit records with unique `id`, `description`, `state` (`waiting` or `resolved`) and one pinned `evidence` file | +| `plan_sha256` | Exact intended comparison freeze, including revisions, model/settings, limits and frozen input identities | +| `authorization` | Optional pinned, explicit authorization evidence for that plan; never inferred from a merged PR or a budget value | + +File hashes establish byte identity, not who wrote or approved a document. The +caller must select authentic owner records and accepted criteria. A third-party +comment, arbitrary file or agent-generated status is not authority merely +because it has a hash. This tool does not introduce an authorization-receipt +service; the existing execution path still requires its exact-plan authorization. + +An optional authorization evidence file has this shape: + +```json +{ + "plan_sha256": "", + "authorized_by": "", + "reference": "" +} +``` + +A criterion acceptance file has exactly these fields: + +```json +{ + "goal_id": "", + "criterion_id": "", + "criterion_sha256": "", + "accepted_by": "", + "decision": "accepted" +} +``` + +Use `tools.goal_checkpoint.criterion_sha256(goal, criterion)` to compute the +binding before recording acceptance. It covers goal ID, pinned accepted scope, +intended plan, criterion ID/description and its evidence references. Changing +any of these needs fresh acceptance evidence. The acceptance pointer itself is +excluded so the record has no circular digest. Accepting a negative comparison +is possible when the goal asks for reliable evidence; the tool never changes +that comparison's grades or paired outcomes. + +## Read, decide and save + +```bash +python -m tools.goal_checkpoint --goal --comparison +python -m tools.goal_checkpoint --goal --comparison \ + --output +python -m tools.goal_checkpoint --goal --comparison \ + --format json --output +``` + +Without `--output`, the command only prints the view. Explicit output must be a +new file outside the comparison directory; existing files are never overwritten. +Keep the goal file and owner/evidence records as sources, not edits to generated +views. JSON and Markdown are alternative projections; no parallel histories are +required. Exit 0 means the view was built, including a stop/repair recommendation. +Malformed goal input or an output error exits 2. An unreadable comparison or +missing/stale evidence appears as `repair_evidence`, with reason codes and +source identities, rather than becoming execution permission. + +`build_checkpoint(Path(goal), Path(comparison))` returns the derived facts and +typed `Recommendation` fields. `recommend(checkpoint)` applies ordered rules: + +| Facts | Action | +| --- | --- | +| Changed/missing evidence, unbound acceptance, stale plan, corrupt receipt, interrupted or ambiguous attempt | `repair_evidence` | +| Completed comparison and every criterion explicitly accepted, with no waiting dependency | `report_accepted` | +| Recorded time/token limit reached | `stop_budget` | +| Other stopped attempt | `prepare_new_plan`; never retry that output directory | +| Fresh plan and every criterion explicitly accepted without requiring a run | `report_accepted`; only the stated goal is accepted | +| Explicit unresolved dependency | `wait_dependency`, naming the dependency ID | +| Completed comparison with remaining/unknown acceptance | `wait_acceptance` | +| Fresh plan without matching recorded authorization | `request_authorization` | +| Fresh plan with matching recorded authorization | `propose_step` through the existing authorized path | + +Observed tokens are a lower bound when provider usage is incomplete. Missing +input/output token totals and cost stay `null`; no reporting path debits a +budget. The threshold stops new trials in the existing controller and is not a +provider-side hard spending cap. A recommendation never executes anything. + +## Before / after example + +The deterministic regression fixture uses the existing comparison test client +and real local subprocess/receipt paths, with **zero model calls**. These are +software-contract fixtures, not measured agent effectiveness or paid-pilot +results. Run `python -m pytest tests/test_goal_checkpoint.py` to verify them. + +Before: the comparison is recorded as completed, and a merged-code record is +available, but the goal owner has not accepted its criterion. A returning agent +must not turn either fact into goal completion. The generated view says: + +```text +Goal: offline-goal +Acceptance owner: unknown +Next action: wait_acceptance +Reasons: criteria_not_accepted, comparison_already_executed +comparison | Owner accepts the evidence | remaining / unknown +``` + +After: the explicit acceptance owner and their pinned, criterion-bound +acceptance record are supplied. The underlying comparison grades and usage are +unchanged. Rebuilding says: + +```text +Goal: offline-goal +Acceptance owner: Fixture Owner +Next action: report_accepted +Reasons: all_criteria_explicitly_accepted +comparison | Owner accepts the evidence | accepted +``` + +Both views also carry the proposed step, dependency facts, frozen revisions and +model/settings, remaining criteria, usage, reason codes and evidence digests. +The agent can recover those facts from the view without earlier chat history. +A changed requirement, missing receipt or interrupted attempt instead produces +`repair_evidence`, even if an acceptance pointer remains present. diff --git a/docs/operations.md b/docs/operations.md index 89af098b..53a426ac 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -28,6 +28,7 @@ Effect classes used below: | Execute/inspect experiments | CLI `run`; `status`; `report` | Experiment-executing / read-only / report-generating | `run` executes caller-supplied commands and records evidence; `status` reads recorded progress; `report` generates HTML from recorded evidence | | Evaluate selected skill revisions | [Bounded skill-evaluation driver](skill-evaluation.md) | Experiment-executing | Explicitly authorized real Codex calls, independent task verdicts and private evidence; separate from numeric engine-performance scoring | | Compare skill additions or updates | [Configured skill comparison](skill-eval-comparison.md), `python -m tools.skill_eval_compare` | Read-only preparation / experiment-executing / report-generating | Fixed Git snapshots and one shared exam; explicit plan authorization for execution; JSON + Markdown regeneration makes no model calls | +| Recover goal progress and the next proposal | [Offline goal checkpoints](goal-checkpoints.md), `python -m tools.goal_checkpoint` | Read-only | Explicit scope/criteria, pinned owner decisions and existing comparison receipts; a disposable view, never execution or acceptance authority | The `collect`, `read` and `resource` commands share the reader's rules; they are distinct from `knowledge search`, whose package catalog filters diff --git a/tests/test_goal_checkpoint.py b/tests/test_goal_checkpoint.py new file mode 100644 index 00000000..7879b3b0 --- /dev/null +++ b/tests/test_goal_checkpoint.py @@ -0,0 +1,290 @@ +import copy +import json +import shutil +import subprocess + +import pytest + +from inferrsi.infra.artifacts import sha256, write_json +from tests.fixtures.skill_eval_compare_support import FixedOutputClient, configuration +from tools import goal_checkpoint as checkpoint +from tools import skill_eval_compare as compare + + +@pytest.fixture(scope="module") +def comparison_templates(tmp_path_factory): + root = tmp_path_factory.mktemp("checkpoint") + try: + compare.require_git_version(subprocess.check_output(("git", "--version"), text=True)) + except ValueError as error: + pytest.skip(str(error)) + config, repo = configuration(root, ids=("one",)) + fresh = root / "fresh" + plan_hash = compare.prepare(config, repo, fresh) + completed = root / "completed" + shutil.copytree(fresh, completed) + client = FixedOutputClient(root) + compare.execute(completed, plan_hash, client=client) + assert len(client.calls) == 2 + return fresh, completed + + +def pin(path, root): + return {"path": path.relative_to(root).as_posix(), "sha256": sha256(path)} + + +def setup_goal(root, template): + run = root / "comparison" + shutil.copytree(template, run) + scope = root / "scope.md" + scope.write_text("Explicit fixture: verify the comparison and obtain owner acceptance.\n", + encoding="utf-8") + goal = {"schema_version": 1, "id": "offline-goal", "title": "Comparison acceptance", + "scope": pin(scope, root), "acceptance_owner": "Fixture Owner", + "criteria": [{"id": "comparison", "description": "Owner accepts the evidence", + "evidence": [], "acceptance": None}], + "dependencies": [], "next_step": "Propose the frozen comparison to the owner", + "plan_sha256": sha256(run / "plan/freeze.json"), "authorization": None} + path = root / "goal.json" + write_json(path, goal) + return path, goal, run + + +def authorize(path, goal): + record = path.parent / "authorization.json" + write_json(record, {"plan_sha256": goal["plan_sha256"], "authorized_by": "Fixture Owner", + "reference": "https://example.invalid/explicit-authorization"}) + goal["authorization"] = pin(record, path.parent) + write_json(path, goal) + + +def accept(path, goal, run): + criterion = goal["criteria"][0] + criterion["evidence"] = [pin(run / "results/status.json", path.parent)] + record = path.parent / "acceptance.json" + write_json(record, {"goal_id": goal["id"], "criterion_id": criterion["id"], + "criterion_sha256": checkpoint.criterion_sha256(goal, criterion), + "accepted_by": goal["acceptance_owner"], "decision": "accepted"}) + criterion["acceptance"] = pin(record, path.parent) + write_json(path, goal) + + +def files(root): + return {p.relative_to(root).as_posix(): (p.read_bytes(), p.stat().st_mtime_ns) + for p in root.rglob("*") if p.is_file()} + + +def test_read_only_deterministic_continuation_and_no_report_writes(tmp_path, comparison_templates): + path, goal, run = setup_goal(tmp_path, comparison_templates[0]) + authorize(path, goal) + before = files(tmp_path) + first = checkpoint.build_checkpoint(path, run) + second = checkpoint.build_checkpoint(path, run) + assert first == second + assert first["recommendation"]["action"] == "propose_step" + assert first["comparison"]["conditions"]["baseline"]["input_tokens"] is None + assert first["comparison"]["status"] == {"execution_state": "not_completed"} + assert first["recommendation"]["input_sha256"] == first["input_sha256"] + view = checkpoint.markdown(first) + assert "propose_step" in view and goal["next_step"] in view + assert "remaining / unknown" in view and "plan_sha256" in view + assert "authorization.json@" in view and "unknown" in view + assert files(tmp_path) == before + assert not (run / "report.json").exists() + assert not (run / "report.md").exists() + + +def test_missing_or_changed_authorization_is_not_permission(tmp_path, comparison_templates): + path, goal, run = setup_goal(tmp_path, comparison_templates[0]) + assert checkpoint.build_checkpoint(path, run)["recommendation"]["action"] == ( + "request_authorization") + authorize(path, goal) + record = path.parent / "authorization.json" + value = json.loads(record.read_bytes()) + value["plan_sha256"] = "0" * 64 + write_json(record, value) + goal["authorization"] = pin(record, path.parent) + write_json(path, goal) + result = checkpoint.build_checkpoint(path, run) + assert not result["authorized"] + assert result["recommendation"]["action"] == "request_authorization" + + +def test_merged_code_and_unknown_acceptance_remain_incomplete(tmp_path, comparison_templates): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + merged = tmp_path / "merged.json" + write_json(merged, {"state": "merged", "checks": "passed"}) + goal["criteria"][0]["evidence"] = [pin(merged, tmp_path)] + goal["acceptance_owner"] = None + write_json(path, goal) + result = checkpoint.build_checkpoint(path, run) + assert result["remaining"] == ["comparison"] + assert result["recommendation"]["action"] == "wait_acceptance" + assert "Acceptance owner: unknown" in checkpoint.markdown(result) + + +def test_explicit_accepted_completion_preserves_paired_semantics(tmp_path, comparison_templates): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + original = compare.read_report(run) + before = files(run) + accept(path, goal, run) + result = checkpoint.build_checkpoint(path, run) + assert result["recommendation"]["action"] == "report_accepted" + assert not result["remaining"] + assert result["comparison"]["counts"] == original["counts"] + assert result["comparison"]["conditions"] == original["conditions"] + assert compare.read_report(run) == original + assert files(run) == before + + +@pytest.mark.parametrize("change", ["scope", "description", "evidence", "owner"]) +def test_acceptance_is_bound_to_criteria_scope_evidence_and_owner( + tmp_path, comparison_templates, change, +): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + accept(path, goal, run) + if change == "scope": + scope = tmp_path / "scope.md" + scope.write_text("Different accepted scope", encoding="utf-8") + goal["scope"] = pin(scope, tmp_path) + elif change == "description": + goal["criteria"][0]["description"] = "A changed requirement" + elif change == "evidence": + goal["criteria"][0]["evidence"] = [pin(run / "report.json", tmp_path)] + else: + goal["acceptance_owner"] = "Another Owner" + write_json(path, goal) + result = checkpoint.build_checkpoint(path, run) + assert result["recommendation"]["action"] == "repair_evidence" + assert "unbound_acceptance:comparison" in result["problems"] + + +@pytest.mark.parametrize("artifact", ["plan/tasks.json", "plan/freeze.json", + "results/status.receipt.json", + "results/one-candidate-1/receipt.json", + "results/one-candidate-1/answer.json"]) +@pytest.mark.parametrize("operation", ["delete", "corrupt"]) +def test_stale_or_missing_evidence_cannot_be_proposed_or_accepted( + tmp_path, comparison_templates, artifact, operation, +): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + accept(path, goal, run) + target = run / artifact + if operation == "delete": + target.unlink() + else: + target.write_text("{broken", encoding="utf-8") + before = files(tmp_path) + result = checkpoint.build_checkpoint(path, run) + assert result["recommendation"]["action"] == "repair_evidence" + assert result["problems"] + assert files(tmp_path) == before + + +def test_changed_plan_settings_require_fresh_authorization(tmp_path, comparison_templates): + path, goal, run = setup_goal(tmp_path, comparison_templates[0]) + authorize(path, goal) + plan = run / "plan/freeze.json" + value = json.loads(plan.read_bytes()) + value["model"]["effort"] = "changed-setting" + write_json(plan, value) + assert "stale_plan" in checkpoint.build_checkpoint(path, run)["problems"] + goal["plan_sha256"] = sha256(plan) + write_json(path, goal) + result = checkpoint.build_checkpoint(path, run) + assert not result["authorized"] + assert result["recommendation"]["action"] == "request_authorization" + + +def test_interruption_never_becomes_a_fresh_or_accepted_attempt(tmp_path, comparison_templates): + path, goal, run = setup_goal(tmp_path, comparison_templates[0]) + authorize(path, goal) + (run / "results/one-baseline-1").mkdir() + write_json(run / "results/start.json", {"started_at": 1}) + result = checkpoint.build_checkpoint(path, run) + assert result["recommendation"]["action"] == "repair_evidence" + assert "interrupted_or_ambiguous_attempt" in result["problems"] + + +@pytest.mark.parametrize("reason", ["token_budget", "time_budget", "client_failed"]) +def test_stopped_attempt_does_not_retry(tmp_path, comparison_templates, reason): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + status_path = run / "results/status.json" + status = json.loads(status_path.read_bytes()) + status.update(execution_state="stopped", stop_reason=reason) + write_json(status_path, status) + write_json(run / "results/status.receipt.json", {"result_sha256": sha256(status_path)}) + result = checkpoint.build_checkpoint(path, run) + expected = "prepare_new_plan" if reason == "client_failed" else "stop_budget" + assert result["recommendation"]["action"] == expected + + +def test_dependency_is_named_and_completion_does_not_hide_it(tmp_path, comparison_templates): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + accept(path, goal, run) + dependency = tmp_path / "review.md" + dependency.write_text("Independent review is still pending", encoding="utf-8") + goal["dependencies"] = [{"id": "review", "description": "Independent review", + "state": "waiting", "evidence": pin(dependency, tmp_path)}] + write_json(path, goal) + result = checkpoint.build_checkpoint(path, run) + assert result["recommendation"]["action"] == "wait_dependency" + assert result["recommendation"]["reasons"] == ("dependency:review",) + assert "Independent review (waiting)" in checkpoint.markdown(result) + + +@pytest.mark.parametrize("usage", [None, "unknown", -1]) +def test_unknown_or_invalid_status_usage_cannot_become_zero( + tmp_path, comparison_templates, usage, +): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + accept(path, goal, run) + status_path = run / "results/status.json" + value = json.loads(status_path.read_bytes()) + value["observed_tokens"] = usage + write_json(status_path, value) + write_json(run / "results/status.receipt.json", {"result_sha256": sha256(status_path)}) + goal["criteria"][0]["acceptance"] = None + goal["criteria"][0]["evidence"] = [] + write_json(path, goal) + result = checkpoint.build_checkpoint(path, run) + assert "invalid_status:observed_tokens" in result["problems"] + assert result["recommendation"]["action"] == "repair_evidence" + assert result["comparison"]["status"]["observed_tokens"] == usage + + +def test_cli_saves_complete_view_and_refuses_input_overwrite( + tmp_path, comparison_templates, capsys, +): + path, goal, run = setup_goal(tmp_path, comparison_templates[0]) + arguments = ["--goal", str(path), "--comparison", str(run)] + assert checkpoint.main(arguments) == 0 + assert "request_authorization" in capsys.readouterr().out + destination = tmp_path / "checkpoint.json" + assert checkpoint.main([*arguments, "--format", "json", "--output", str(destination)]) == 0 + saved = json.loads(destination.read_bytes()) + assert saved["input_sha256"] == checkpoint.build_checkpoint(path, run)["input_sha256"] + before = path.read_bytes() + with pytest.raises(SystemExit) as error: + checkpoint.main([*arguments, "--output", str(path)]) + assert error.value.code == 2 + assert path.read_bytes() == before + with pytest.raises(SystemExit): + checkpoint.main([*arguments, "--output", str(run / "plan/checkpoint.md")]) + assert not (run / "plan/checkpoint.md").exists() + + +@pytest.mark.parametrize("change", ["empty", "duplicate", "unknown", "invalid_pin"]) +def test_invalid_explicit_goal_contract_fails_at_boundary(tmp_path, comparison_templates, change): + path, goal, run = setup_goal(tmp_path, comparison_templates[0]) + if change == "empty": + goal["criteria"] = [] + elif change == "duplicate": + goal["criteria"].append(copy.deepcopy(goal["criteria"][0])) + elif change == "unknown": + goal["approve_everything"] = True + else: + goal["scope"]["sha256"] = "not-a-digest" + write_json(path, goal) + with pytest.raises(ValueError): + checkpoint.build_checkpoint(path, run) diff --git a/tools/goal_checkpoint.py b/tools/goal_checkpoint.py new file mode 100644 index 00000000..632b4ac1 --- /dev/null +++ b/tools/goal_checkpoint.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Literal + +from inferrsi.infra.artifacts import digest, sha256 +from tools.skill_eval_compare import ROLES, read_report + +Action = Literal[ + "repair_evidence", "prepare_new_plan", "stop_budget", "wait_dependency", + "wait_acceptance", "request_authorization", "propose_step", "report_accepted", +] + + +@dataclass(frozen=True) +class Recommendation: + action: Action + reasons: tuple[str, ...] + evidence: tuple[str, ...] + input_sha256: str + + +def _fields(value, required: set[str], optional: set[str] = frozenset()) -> None: + if not isinstance(value, dict) or required - value.keys() or ( + value.keys() - required - optional + ): + raise ValueError("Missing or unsupported checkpoint input fields") + + +def _text(value) -> None: + if not isinstance(value, str) or not value.strip(): + raise ValueError("Checkpoint text fields must be nonempty strings") + + +def _hash(value) -> None: + if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value): + raise ValueError("Expected a lowercase SHA-256 digest") + + +def _pin(value) -> None: + _fields(value, {"path", "sha256"}) + _text(value["path"]) + _hash(value["sha256"]) + if Path(value["path"]).is_absolute(): + raise ValueError("Evidence paths must be relative to the goal input") + + +def _validate(goal: dict) -> None: + _fields(goal, {"schema_version", "id", "title", "scope", "acceptance_owner", "criteria", + "dependencies", "next_step", "plan_sha256", "authorization"}) + if type(goal["schema_version"]) is not int or goal["schema_version"] != 1: + raise ValueError("Unsupported goal checkpoint schema") + for name in ("id", "title", "next_step"): + _text(goal[name]) + _hash(goal["plan_sha256"]) + _pin(goal["scope"]) + if goal["acceptance_owner"] is not None: + _text(goal["acceptance_owner"]) + if not isinstance(goal["criteria"], list) or not goal["criteria"]: + raise ValueError("Explicit nonempty accepted criteria are required") + for criterion in goal["criteria"]: + _fields(criterion, {"id", "description", "evidence", "acceptance"}) + _text(criterion["id"]) + _text(criterion["description"]) + if not isinstance(criterion["evidence"], list): + raise ValueError("Criterion evidence must be a list of pinned files") + for pin in criterion["evidence"]: + _pin(pin) + if criterion["acceptance"] is not None: + _pin(criterion["acceptance"]) + if len({c["id"] for c in goal["criteria"]}) != len(goal["criteria"]): + raise ValueError("Criterion IDs must be unique") + if not isinstance(goal["dependencies"], list): + raise ValueError("Dependencies must be an explicit list") + for dependency in goal["dependencies"]: + _fields(dependency, {"id", "description", "state", "evidence"}) + _text(dependency["id"]) + _text(dependency["description"]) + if dependency["state"] not in ("waiting", "resolved"): + raise ValueError("Dependency state must be waiting or resolved") + _pin(dependency["evidence"]) + if len({d["id"] for d in goal["dependencies"]}) != len(goal["dependencies"]): + raise ValueError("Dependency IDs must be unique") + if goal["authorization"] is not None: + _pin(goal["authorization"]) + + +def criterion_sha256(goal: dict, criterion: dict) -> str: + return digest({"goal_id": goal["id"], "scope": goal["scope"], + "plan_sha256": goal["plan_sha256"], + "criterion": {k: criterion[k] for k in ("id", "description", "evidence")}}) + + +def recommend(checkpoint: dict) -> Recommendation: + def decision(action: Action, *reasons: str) -> Recommendation: + return Recommendation(action, reasons, tuple(checkpoint["sources"]), + checkpoint["input_sha256"]) + + if checkpoint["problems"]: + return decision("repair_evidence", *checkpoint["problems"]) + comparison = checkpoint["comparison"] + status = comparison["status"] + state = status["execution_state"] + if state == "completed" and not checkpoint["remaining"] and not checkpoint["waiting"]: + return decision("report_accepted", "all_criteria_explicitly_accepted") + if status.get("stop_reason") in ("token_budget", "time_budget"): + return decision("stop_budget", "declared_limit_reached") + if (status.get("observed_tokens") is not None + and status["observed_tokens"] >= comparison["limits"]["token_stop_new_trials"]): + return decision("stop_budget", "observed_token_limit_reached") + if state == "stopped": + return decision("prepare_new_plan", "attempt_stopped_no_resume") + if not checkpoint["remaining"] and not checkpoint["waiting"]: + return decision("report_accepted", "all_criteria_explicitly_accepted") + if checkpoint["waiting"]: + return decision("wait_dependency", *("dependency:" + d for d in checkpoint["waiting"])) + if state == "completed": + return decision("wait_acceptance", "criteria_not_accepted", "comparison_already_executed") + if not checkpoint["authorized"]: + return decision("request_authorization", "exact_plan_authorization_missing") + return decision("propose_step", "fresh_plan_and_recorded_authorization") + + +def build_checkpoint(goal_path: Path, comparison_dir: Path) -> dict: + raw_goal = goal_path.read_bytes() + goal = json.loads(raw_goal) + _validate(goal) + sources = {"goal": {"sha256": hashlib.sha256(raw_goal).hexdigest()}} + problems = [] + cache = {} + + def read(pin: dict) -> bytes | None: + key = (pin["path"], pin["sha256"]) + if key not in cache: + try: + data = (goal_path.parent / pin["path"]).read_bytes() + actual = hashlib.sha256(data).hexdigest() + except OSError: + data, actual = None, None + sources[pin["path"] + "@" + pin["sha256"]] = { + "expected_sha256": pin["sha256"], "actual_sha256": actual} + cache[key] = data if actual == pin["sha256"] else None + if cache[key] is None: + problems.append("missing_or_changed:" + pin["path"]) + return cache[key] + + def record(pin: dict) -> dict | None: + data = read(pin) + if data is None: + return None + try: + value = json.loads(data) + except ValueError: + value = None + if not isinstance(value, dict): + problems.append("invalid_record:" + pin["path"]) + return None + return value + + read(goal["scope"]) + criteria = [] + for criterion in goal["criteria"]: + complete = bool(criterion["evidence"]) + for pin in criterion["evidence"]: + complete = read(pin) is not None and complete + acceptance = record(criterion["acceptance"]) if criterion["acceptance"] else None + accepted = False + if acceptance is not None: + expected = {"goal_id": goal["id"], "criterion_id": criterion["id"], + "criterion_sha256": criterion_sha256(goal, criterion), + "accepted_by": goal["acceptance_owner"], "decision": "accepted"} + accepted = complete and goal["acceptance_owner"] is not None and acceptance == expected + if not accepted: + problems.append("unbound_acceptance:" + criterion["id"]) + criteria.append({**criterion, "accepted": accepted}) + waiting = [] + for dependency in goal["dependencies"]: + read(dependency["evidence"]) + if dependency["state"] == "waiting": + waiting.append(dependency["id"]) + authorization = record(goal["authorization"]) if goal["authorization"] else None + authorized = False + if authorization is not None: + _fields(authorization, {"plan_sha256", "authorized_by", "reference"}) + _text(authorization["authorized_by"]) + _text(authorization["reference"]) + _hash(authorization["plan_sha256"]) + authorized = authorization["plan_sha256"] == goal["plan_sha256"] + + comparison = None + sources["comparison"] = {"artifacts": {}} + artifact_names = {"plan/freeze.json", "results/start.json", "results/status.json", + "results/status.receipt.json"} + try: + report = read_report(comparison_dir) + sources["comparison"].update(plan_sha256=report["plan_sha256"], + report_sha256=digest(report)) + if report["plan_sha256"] != goal["plan_sha256"]: + problems.append("stale_plan") + trials = [arm for pair in report["pairs"] for arm in (pair[r] for r in ROLES)] + trials.extend(report["discovery"]) + for trial in trials: + artifact_names.update(trial["evidence"].values()) + problems.extend("trial:" + t["task_id"] + ":" + t["condition"] + ":" + + t["evaluation_error"] for t in trials if t.get("evaluation_error")) + status = report["status"] + state = status["execution_state"] + if state == "unverified": + problems.append(status["evaluation_error"]) + elif state == "not_completed": + if any((comparison_dir / "results").iterdir()): + problems.append("interrupted_or_ambiguous_attempt") + elif state in ("completed", "stopped"): + for name in ("recorded_trials", "expected_trials", "observed_tokens"): + if type(status[name]) is not int or status[name] < 0: + problems.append("invalid_status:" + name) + if state == "stopped" and not isinstance(status["stop_reason"], str): + problems.append("missing_stop_reason") + recorded = sum(not t["missing"] for t in trials) + if status["recorded_trials"] != recorded or status["expected_trials"] != len(trials): + problems.append("inconsistent_trial_counts") + if state == "completed" and (recorded != len(trials) or status["stop_reason"]): + problems.append("incomplete_or_stopped_attempt") + else: + problems.append("unknown_execution_state") + comparison = {name: report[name] for name in ( + "plan_sha256", "model", "limits", "status", "counts", "conditions", + "discovery_conditions")} + comparison["revisions"] = {r: report["arms"][r]["git_revision"] for r in ROLES} + except (OSError, ValueError, KeyError, TypeError, AttributeError): + problems.append("comparison_unreadable_or_stale") + for name in sorted(artifact_names): + path = comparison_dir / name + try: + actual = sha256(path) if path.is_file() else None + except OSError: + actual = None + problems.append("unreadable_artifact:" + name) + sources["comparison"]["artifacts"][name] = actual + + checkpoint = {"schema_version": 1, "id": goal["id"], "title": goal["title"], + "acceptance_owner": goal["acceptance_owner"], "criteria": criteria, + "remaining": [c["id"] for c in criteria if not c["accepted"]], + "dependencies": goal["dependencies"], "waiting": waiting, + "next_step": goal["next_step"], "authorized": authorized, + "comparison": comparison, "sources": sources, + "problems": sorted(set(problems))} + checkpoint["input_sha256"] = digest(checkpoint) + checkpoint["recommendation"] = asdict(recommend(checkpoint)) + return checkpoint + + +def markdown(checkpoint: dict) -> str: + def text(value) -> str: + return str(value).replace("\n", " ").replace("\r", " ").replace("|", "\\|") + + recommendation = checkpoint["recommendation"] + lines = [f"# Goal checkpoint: {text(checkpoint['title'])}", "", + f"Goal: {text(checkpoint['id'])}", + f"Acceptance owner: {text(checkpoint['acceptance_owner'] or 'unknown')}", + f"Input SHA-256: `{checkpoint['input_sha256']}`", "", + f"Next action: **{recommendation['action']}**", + "Reasons: " + ", ".join(text(r) for r in recommendation["reasons"]), + f"Proposed step: {text(checkpoint['next_step'])}", "", + "| Criterion | Requirement | Acceptance |", + "| --- | --- | --- |"] + lines.extend(f"| {text(c['id'])} | {text(c['description'])} | " + + ("accepted" if c["accepted"] else "remaining / unknown") + " |" + for c in checkpoint["criteria"]) + lines.extend(["", "Dependencies:"]) + lines.extend(f"- {text(d['id'])}: {text(d['description'])} ({d['state']})" + for d in checkpoint["dependencies"]) + if not checkpoint["dependencies"]: + lines.append("- None declared.") + if checkpoint["comparison"] is not None: + lines.extend(["", "Comparison facts (existing verdicts and usage unchanged):", "```json", + json.dumps(checkpoint["comparison"], sort_keys=True, indent=2, + ensure_ascii=False, allow_nan=False), "```"]) + lines.extend(["", "Evidence identities:"]) + lines.extend(f"- {text(name)}: `{json.dumps(value, sort_keys=True)}`" + for name, value in checkpoint["sources"].items()) + lines.extend(["", "Recommendation only. This view cannot execute, resume, authorize or " + "accept work. Unknown usage remains unknown. Owner records are supplied " + "explicitly; file digests verify bytes, not the signer's identity.", ""]) + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Rebuild an offline goal checkpoint") + parser.add_argument("--goal", type=Path, required=True) + parser.add_argument("--comparison", type=Path, required=True) + parser.add_argument("--format", choices=("markdown", "json"), default="markdown") + parser.add_argument("--output", type=Path) + args = parser.parse_args(argv) + try: + checkpoint = build_checkpoint(args.goal, args.comparison) + output = markdown(checkpoint) if args.format == "markdown" else ( + json.dumps(checkpoint, sort_keys=True, ensure_ascii=False, indent=2, + allow_nan=False) + "\n") + if args.output is None: + print(output, end="") + else: + # Derived views must never overwrite their inputs or retained evidence. + if args.output.resolve().is_relative_to(args.comparison.resolve()): + raise ValueError("Checkpoint output must be outside the comparison evidence") + with args.output.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(output) + except (OSError, ValueError) as exc: + parser.error(str(exc)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index e7f640d6..1311eed0 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -520,7 +520,7 @@ def condition_summary(trials: list[dict]) -> dict: return groups -def report(output: Path) -> dict: +def read_report(output: Path) -> dict: plan = load_plan(output) trials = [retained_trial(output, entry) for entry in plan["schedule"]] index = {(t["task_id"], t["repeat"], t["condition"]): t for t in trials} @@ -569,6 +569,11 @@ def report(output: Path) -> dict: "claim": "Pipeline results only; fixed-output offline clients do not establish " "skill efficacy. Discovery is separate. Unknown usage and cost remain unknown.", } + return result + + +def report(output: Path) -> dict: + result = read_report(output) write_json(output / "report.json", result) (output / "report.md").write_text(markdown(result), encoding="utf-8") return result From 21efe995adf9965172760d4b0aa7cf7c4e913892 Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Sun, 25 Oct 2026 22:25:26 +0800 Subject: [PATCH 2/3] fix: preserve execution failures in checkpoint decisions Separate retained-artifact integrity from authentic trial errors, and preserve the input-integrity flag even when another error is recorded. Agent-assisted-by: Codex (implementation) Signed-off-by: GUOGUO --- docs/goal-checkpoints.md | 7 +++++ tests/test_goal_checkpoint.py | 49 +++++++++++++++++++++++++++++++++++ tools/goal_checkpoint.py | 26 +++++++++++++++++-- tools/skill_eval_compare.py | 2 +- 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/docs/goal-checkpoints.md b/docs/goal-checkpoints.md index 4da7fac5..885b24dd 100644 --- a/docs/goal-checkpoints.md +++ b/docs/goal-checkpoints.md @@ -179,3 +179,10 @@ model/settings, remaining criteria, usage, reason codes and evidence digests. The agent can recover those facts from the view without earlier chat history. A changed requirement, missing receipt or interrupted attempt instead produces `repair_evidence`, even if an acceptance pointer remains present. + +An authentic client failure/timeout with valid receipts instead remains a +recorded stopped attempt and recommends `prepare_new_plan`. A completed trial's +evaluation failure remains its existing `INCONCLUSIVE` outcome and waits for +owner acceptance; it is not evidence corruption. Changed inputs or damaged +retained artifacts still block acceptance, including when the same trial also +records an execution failure. No recommendation edits the retained failure. diff --git a/tests/test_goal_checkpoint.py b/tests/test_goal_checkpoint.py index 7879b3b0..9d2a9592 100644 --- a/tests/test_goal_checkpoint.py +++ b/tests/test_goal_checkpoint.py @@ -219,6 +219,55 @@ def test_stopped_attempt_does_not_retry(tmp_path, comparison_templates, reason): assert result["recommendation"]["action"] == expected +@pytest.mark.parametrize("mode", ["nonzero", "timeout", "missing_output", "invalid_json", + "invalid_utf8", "markdown_fence", "invalid_trace"]) +def test_real_client_failures_preserve_evidence_and_choose_the_actual_next_step( + tmp_path, comparison_templates, mode, +): + path, goal, run = setup_goal(tmp_path, comparison_templates[0]) + client = FixedOutputClient(tmp_path, {("one", "candidate"): mode}) + report = compare.execute(run, goal["plan_sha256"], client=client) + before = files(run) + result = checkpoint.build_checkpoint(path, run) + assert not result["problems"] + expected = "prepare_new_plan" if mode in ("nonzero", "timeout") else "wait_acceptance" + assert result["recommendation"]["action"] == expected + assert result["comparison"]["counts"] == report["counts"] + assert result["comparison"]["counts"]["inconclusive"] == 1 + assert files(run) == before + assert len(client.calls) == 2 + + +def test_execution_failure_cannot_hide_damaged_retained_evidence(tmp_path, comparison_templates): + path, goal, run = setup_goal(tmp_path, comparison_templates[0]) + client = FixedOutputClient(tmp_path, {("one", "candidate"): "nonzero"}) + compare.execute(run, goal["plan_sha256"], client=client) + assert checkpoint.build_checkpoint(path, run)["recommendation"]["action"] == "prepare_new_plan" + (run / "results/one-candidate-1/execution.json").write_text("{}", encoding="utf-8") + result = checkpoint.build_checkpoint(path, run) + assert result["recommendation"]["action"] == "repair_evidence" + assert any("retained_trial_integrity_mismatch" in reason for reason in result["problems"]) + + +def test_retained_input_change_flag_cannot_be_hidden_by_another_error( + tmp_path, comparison_templates, +): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + result_path = run / "results/one-candidate-1/result.json" + record = json.loads(result_path.read_bytes()) + record.update(inputs_unchanged=False, evaluation_error="agent_trace_error") + write_json(result_path, record) + write_json(result_path.parent / "receipt.json", {"result_sha256": sha256(result_path)}) + accepted_report = compare.read_report(run) + trial = accepted_report["pairs"][0]["candidate"] + assert trial["evaluation_error"] == "agent_trace_error" + assert trial["inputs_unchanged"] is False + accept(path, goal, run) + result = checkpoint.build_checkpoint(path, run) + assert result["recommendation"]["action"] == "repair_evidence" + assert any("inputs_changed" in reason for reason in result["problems"]) + + def test_dependency_is_named_and_completion_does_not_hide_it(tmp_path, comparison_templates): path, goal, run = setup_goal(tmp_path, comparison_templates[1]) accept(path, goal, run) diff --git a/tools/goal_checkpoint.py b/tools/goal_checkpoint.py index 632b4ac1..03893fc9 100644 --- a/tools/goal_checkpoint.py +++ b/tools/goal_checkpoint.py @@ -96,6 +96,25 @@ def criterion_sha256(goal: dict, criterion: dict) -> str: "criterion": {k: criterion[k] for k in ("id", "description", "evidence")}}) +def _trial_problem(trial: dict) -> str | None: + error = trial.get("evaluation_error") + if trial.get("inputs_unchanged") is False: + return "inputs_changed" + if error in ("interrupted_attempt", "inputs_changed") or (error and error.startswith( + ("missing_retained_", "invalid_retained_", "retained_") + )): + return error + if trial["missing"]: + return None + expected = trial["evidence_sha256"] + if expected is None or trial["output_sha256"] != trial["retained_output_sha256"] or any( + trial["retained_evidence_sha256"].get(name) != value for name, value in expected.items() + ): + # A recorded execution error must not hide independently damaged evidence. + return "retained_trial_integrity_mismatch" + return None + + def recommend(checkpoint: dict) -> Recommendation: def decision(action: Action, *reasons: str) -> Recommendation: return Recommendation(action, reasons, tuple(checkpoint["sources"]), @@ -206,8 +225,11 @@ def record(pin: dict) -> dict | None: trials.extend(report["discovery"]) for trial in trials: artifact_names.update(trial["evidence"].values()) - problems.extend("trial:" + t["task_id"] + ":" + t["condition"] + ":" - + t["evaluation_error"] for t in trials if t.get("evaluation_error")) + for trial in trials: + problem = _trial_problem(trial) + if problem is not None: + problems.append("trial:" + trial["task_id"] + ":" + trial["condition"] + + ":" + problem) status = report["status"] state = status["execution_state"] if state == "unverified": diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index 1311eed0..1f924d03 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -472,7 +472,7 @@ def retained_trial(output: Path, entry: dict) -> dict: raise ValueError("Result identity disagrees with frozen schedule") # Select report fields; raw commands and private workspace paths stay in local artifacts. selected = {key: record[key] for key in ("task_id", "condition", "repeat", "split", - "verdict", "checks", "elapsed_seconds", "observations")} + "verdict", "checks", "elapsed_seconds", "observations", "inputs_unchanged")} selected["observations"] = {"usage": record["observations"]["usage"]} error = record.get("evaluation_error") if data is None: From f1b8b87064e48750e60353a6d78ac41904abaf3d Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Sun, 25 Oct 2026 22:35:36 +0800 Subject: [PATCH 3/3] fix: retain input identities when checkpoint evidence is damaged Fingerprint the actual comparison inputs and controller bytes even when report verification exits early, preserving distinct repair states. Agent-assisted-by: Codex (implementation) Signed-off-by: GUOGUO --- docs/goal-checkpoints.md | 3 +++ tests/test_goal_checkpoint.py | 24 ++++++++++++++++++++++++ tools/goal_checkpoint.py | 14 +++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/goal-checkpoints.md b/docs/goal-checkpoints.md index 885b24dd..39fdd4f8 100644 --- a/docs/goal-checkpoints.md +++ b/docs/goal-checkpoints.md @@ -179,6 +179,9 @@ model/settings, remaining criteria, usage, reason codes and evidence digests. The agent can recover those facts from the view without earlier chat history. A changed requirement, missing receipt or interrupted attempt instead produces `repair_evidence`, even if an acceptance pointer remains present. +Input identity includes the actual plan/validator/results files and comparison +controller bytes even when report reading fails early, so different damaged +inputs remain distinguishable rather than collapsing into one error identity. An authentic client failure/timeout with valid receipts instead remains a recorded stopped attempt and recommends `prepare_new_plan`. A completed trial's diff --git a/tests/test_goal_checkpoint.py b/tests/test_goal_checkpoint.py index 9d2a9592..f5806b42 100644 --- a/tests/test_goal_checkpoint.py +++ b/tests/test_goal_checkpoint.py @@ -196,6 +196,30 @@ def test_changed_plan_settings_require_fresh_authorization(tmp_path, comparison_ assert result["recommendation"]["action"] == "request_authorization" +@pytest.mark.parametrize("artifact", ["plan/tasks.json", "validator/verify.py", + "results/one-candidate-1/result.json"]) +def test_distinct_damaged_inputs_keep_distinct_checkpoint_identities( + tmp_path, comparison_templates, artifact, +): + path, goal, run = setup_goal(tmp_path, comparison_templates[1]) + target = run / artifact + target.write_text("{first damaged input", encoding="utf-8") + if artifact.endswith("result.json"): + write_json(target.parent / "receipt.json", {"result_sha256": sha256(target)}) + first = checkpoint.build_checkpoint(path, run) + target.write_text("{second damaged input", encoding="utf-8") + if artifact.endswith("result.json"): + write_json(target.parent / "receipt.json", {"result_sha256": sha256(target)}) + second = checkpoint.build_checkpoint(path, run) + assert first["recommendation"]["action"] == second["recommendation"]["action"] == ( + "repair_evidence") + assert first["input_sha256"] != second["input_sha256"] + actual = second["sources"]["comparison"]["artifacts"][artifact] + assert actual == sha256(target) + assert second["sources"]["controllers"] == { + name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS} + + def test_interruption_never_becomes_a_fresh_or_accepted_attempt(tmp_path, comparison_templates): path, goal, run = setup_goal(tmp_path, comparison_templates[0]) authorize(path, goal) diff --git a/tools/goal_checkpoint.py b/tools/goal_checkpoint.py index 03893fc9..dddf8e98 100644 --- a/tools/goal_checkpoint.py +++ b/tools/goal_checkpoint.py @@ -9,7 +9,7 @@ from typing import Literal from inferrsi.infra.artifacts import digest, sha256 -from tools.skill_eval_compare import ROLES, read_report +from tools.skill_eval_compare import CONTROLLERS, ROLES, ROOT, read_report Action = Literal[ "repair_evidence", "prepare_new_plan", "stop_budget", "wait_dependency", @@ -216,6 +216,10 @@ def record(pin: dict) -> dict | None: artifact_names = {"plan/freeze.json", "results/start.json", "results/status.json", "results/status.receipt.json"} try: + # Failed reads still need the identities of the actual damaged inputs. + for folder in ("plan", "validator", "results"): + artifact_names.update(p.relative_to(comparison_dir).as_posix() + for p in (comparison_dir / folder).rglob("*") if p.is_file()) report = read_report(comparison_dir) sources["comparison"].update(plan_sha256=report["plan_sha256"], report_sha256=digest(report)) @@ -264,6 +268,14 @@ def record(pin: dict) -> dict | None: actual = None problems.append("unreadable_artifact:" + name) sources["comparison"]["artifacts"][name] = actual + sources["controllers"] = {} + for name in CONTROLLERS: + try: + actual = sha256(ROOT / name) + except OSError: + actual = None + problems.append("unreadable_controller:" + name) + sources["controllers"][name] = actual checkpoint = {"schema_version": 1, "id": goal["id"], "title": goal["title"], "acceptance_owner": goal["acceptance_owner"], "criteria": criteria,