From 54ca732be68e71fa03a5778e5323819cbc7f7e1e Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Thu, 11 Jun 2026 20:20:14 -0400 Subject: [PATCH] feat: expose sandbox run evidence refs for workroom consumption, add teardown validation (#256) Adds runtime_teardown_complete validation to validate_runtime_sandbox_run.py: parity must be runtime_observed, evidence and receipt refs are required, failure codes must be empty, teardownState must be teardown_complete. Adds runtime-sandbox-run.teardown.valid.json: valid teardown_complete fixture with workroom receipt (receipt://devsecops-workroom/sandbox-evidence/...). Adds runtime-sandbox-run.teardown.missing-evidence.invalid.json: invalid teardown_complete with empty evidenceRefs and receiptRefs. Adds shared-receipt fixture and teardown fixture to VALID_FIXTURES. Validator now covers all 4 states: requested, allocated, failed, and teardown_complete. Wires validate-runtime-sandbox-run into make validate. Closes #256 --- Makefile | 7 ++-- ...run.teardown.missing-evidence.invalid.json | 30 ++++++++++++++++ .../runtime-sandbox-run.teardown.valid.json | 36 +++++++++++++++++++ tools/validate_runtime_sandbox_run.py | 23 +++++++++--- 4 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/sandbox/runtime-sandbox-run.teardown.missing-evidence.invalid.json create mode 100644 tests/fixtures/sandbox/runtime-sandbox-run.teardown.valid.json diff --git a/Makefile b/Makefile index 8856a65..4b6d100 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: validate test validate-agent-cycle-health validate-authority-dependency-evidence validate-prometheus-sr validate-reasoning-failure-traces validate-governance-context validate-lattice-data-governai-execution-refs validate-lattice-runtime-profile-refs validate-network-native-assistant-evidence validate-guardrail-evidence-artifacts validate-stop-gate-evaluator validate-guarded-workcell-artifact validate-guarded-workcell-executor validate-guarded-invocation-artifact validate-guarded-invocation validate-agentic-pr-work-order validate-semantic-enterprise-agent-boundary validate-ops-history-contracts validate-action-contracts validate-agent-operation-contract validate-superconscious-reasoning-import validate-agent-harness-runtime-contracts validate-bounded-action-loop agentplane-evidence-receipt-composition-tier2-binding-ci lawful-learning-phase9-contract-ci validate-evidence-receipt-binding validate-semantic-activation-receipt validate-governed-run-contract validate-preflight-receipt validate-attempt-admission-receipt validate-verification-execution-receipt validate-synthetic-verification-receipt validate-governed-runner-v0-2-contract-chain validate-budget-settlement-receipt validate-rollback-receipts validate-run-dossier validate-governed-runner-readonly validate-workroom-context-evidence validate-wallguard-collaboration-admission validate-prophet-mesh-agentplane-adapter +.PHONY: validate test validate-agent-cycle-health validate-authority-dependency-evidence validate-prometheus-sr validate-reasoning-failure-traces validate-governance-context validate-lattice-data-governai-execution-refs validate-lattice-runtime-profile-refs validate-network-native-assistant-evidence validate-guardrail-evidence-artifacts validate-stop-gate-evaluator validate-guarded-workcell-artifact validate-guarded-workcell-executor validate-guarded-invocation-artifact validate-guarded-invocation validate-agentic-pr-work-order validate-semantic-enterprise-agent-boundary validate-ops-history-contracts validate-action-contracts validate-agent-operation-contract validate-superconscious-reasoning-import validate-agent-harness-runtime-contracts validate-bounded-action-loop agentplane-evidence-receipt-composition-tier2-binding-ci lawful-learning-phase9-contract-ci validate-evidence-receipt-binding validate-semantic-activation-receipt validate-governed-run-contract validate-preflight-receipt validate-attempt-admission-receipt validate-verification-execution-receipt validate-synthetic-verification-receipt validate-governed-runner-v0-2-contract-chain validate-budget-settlement-receipt validate-rollback-receipts validate-run-dossier validate-governed-runner-readonly validate-workroom-context-evidence validate-wallguard-collaboration-admission validate-prophet-mesh-agentplane-adapter validate-runtime-sandbox-run -validate: validate-agent-cycle-health validate-authority-dependency-evidence validate-prometheus-sr validate-reasoning-failure-traces validate-governance-context validate-lattice-data-governai-execution-refs validate-lattice-runtime-profile-refs validate-network-native-assistant-evidence validate-guardrail-evidence-artifacts validate-stop-gate-evaluator validate-guarded-workcell-artifact validate-guarded-workcell-executor validate-guarded-invocation-artifact validate-guarded-invocation validate-agentic-pr-work-order validate-semantic-enterprise-agent-boundary validate-ops-history-contracts validate-action-contracts validate-agent-operation-contract validate-superconscious-reasoning-import validate-agent-harness-runtime-contracts validate-bounded-action-loop agentplane-evidence-receipt-composition-tier2-binding-ci lawful-learning-phase9-contract-ci validate-evidence-receipt-binding validate-semantic-activation-receipt validate-governed-run-contract validate-preflight-receipt validate-attempt-admission-receipt validate-verification-execution-receipt validate-synthetic-verification-receipt validate-governed-runner-v0-2-contract-chain validate-budget-settlement-receipt validate-rollback-receipts validate-run-dossier validate-governed-runner-readonly validate-workroom-context-evidence validate-wallguard-collaboration-admission validate-prophet-mesh-agentplane-adapter +validate: validate-agent-cycle-health validate-authority-dependency-evidence validate-prometheus-sr validate-reasoning-failure-traces validate-governance-context validate-lattice-data-governai-execution-refs validate-lattice-runtime-profile-refs validate-network-native-assistant-evidence validate-guardrail-evidence-artifacts validate-stop-gate-evaluator validate-guarded-workcell-artifact validate-guarded-workcell-executor validate-guarded-invocation-artifact validate-guarded-invocation validate-agentic-pr-work-order validate-semantic-enterprise-agent-boundary validate-ops-history-contracts validate-action-contracts validate-agent-operation-contract validate-superconscious-reasoning-import validate-agent-harness-runtime-contracts validate-bounded-action-loop agentplane-evidence-receipt-composition-tier2-binding-ci lawful-learning-phase9-contract-ci validate-evidence-receipt-binding validate-semantic-activation-receipt validate-governed-run-contract validate-preflight-receipt validate-attempt-admission-receipt validate-verification-execution-receipt validate-synthetic-verification-receipt validate-governed-runner-v0-2-contract-chain validate-budget-settlement-receipt validate-rollback-receipts validate-run-dossier validate-governed-runner-readonly validate-workroom-context-evidence validate-wallguard-collaboration-admission validate-prophet-mesh-agentplane-adapter validate-runtime-sandbox-run python3 tools/validate_execution_timing.py validate-governance-context: @@ -248,6 +248,9 @@ validate-prophet-mesh-agentplane-adapter: python3 -m json.tool contracts/prophet-mesh/prophet-mesh-agentplane-adapter.v0.1.json >/dev/null python3 tools/validate_prophet_mesh_agentplane_adapter.py +validate-runtime-sandbox-run: + python3 tools/validate_runtime_sandbox_run.py + validate-agent-cycle-health: python3 tools/validate_agent_cycle_health.py diff --git a/tests/fixtures/sandbox/runtime-sandbox-run.teardown.missing-evidence.invalid.json b/tests/fixtures/sandbox/runtime-sandbox-run.teardown.missing-evidence.invalid.json new file mode 100644 index 0000000..536898a --- /dev/null +++ b/tests/fixtures/sandbox/runtime-sandbox-run.teardown.missing-evidence.invalid.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": "0.1.0", + "runtimeRunId": "agentplane:runtime-sandbox-run:teardown:missing-ev-example", + "requestRef": "environment:validate-change-v2-request:scope-d-missing-ev", + "executorPlane": "AgentPlane", + "executionMode": "runtime_contract", + "runtimeParityLevel": "runtime_observed", + "runStatus": "runtime_teardown_complete", + "environmentRef": "environment://runtime/scope-d-missing-ev/teardown", + "baselineRef": "workspace://scope-d/main", + "changedServiceRefs": [ + "service://scope-d/api" + ], + "dependencyGraphRef": "dependency-graph://runtime/scope-d-missing-ev/teardown", + "routingRef": "routing://runtime/scope-d-missing-ev/teardown", + "isolationRefs": { + "network": "isolation://runtime/scope-d-missing-ev/network/teardown", + "async": "isolation://runtime/scope-d-missing-ev/async/teardown", + "stateful": "isolation://runtime/scope-d-missing-ev/stateful/teardown" + }, + "evidenceRefs": [], + "receiptRefs": [], + "failureCodes": [], + "teardownState": "teardown_complete", + "leakCheckRef": "leak-check://runtime/scope-d-missing-ev/teardown-complete", + "issuedAt": "2026-06-11T14:30:00Z", + "nonClaims": [ + "Missing evidence refs is the invalid condition under test." + ] +} diff --git a/tests/fixtures/sandbox/runtime-sandbox-run.teardown.valid.json b/tests/fixtures/sandbox/runtime-sandbox-run.teardown.valid.json new file mode 100644 index 0000000..4493e31 --- /dev/null +++ b/tests/fixtures/sandbox/runtime-sandbox-run.teardown.valid.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": "0.1.0", + "runtimeRunId": "agentplane:runtime-sandbox-run:teardown:scope-d-example", + "requestRef": "environment:validate-change-v2-request:scope-d-example", + "executorPlane": "AgentPlane", + "executionMode": "runtime_contract", + "runtimeParityLevel": "runtime_observed", + "runStatus": "runtime_teardown_complete", + "environmentRef": "environment://runtime/scope-d-example/teardown", + "baselineRef": "workspace://scope-d/main", + "changedServiceRefs": [ + "service://scope-d/api" + ], + "dependencyGraphRef": "dependency-graph://runtime/scope-d-example/teardown", + "routingRef": "routing://runtime/scope-d-example/teardown", + "isolationRefs": { + "network": "isolation://runtime/scope-d-example/network/teardown", + "async": "isolation://runtime/scope-d-example/async/teardown", + "stateful": "isolation://runtime/scope-d-example/stateful/teardown" + }, + "evidenceRefs": [ + "evidence://agentplane/runtime-sandbox-run/scope-d-example/teardown" + ], + "receiptRefs": [ + "receipt://devsecops-workroom/sandbox-evidence/scope-d-example/teardown-001" + ], + "failureCodes": [], + "teardownState": "teardown_complete", + "leakCheckRef": "leak-check://runtime/scope-d-example/teardown-complete", + "issuedAt": "2026-06-11T14:00:00Z", + "nonClaims": [ + "Teardown complete does not certify full Signadot runtime parity.", + "Teardown complete does not imply leak-free outcome without inspecting leakCheckRef.", + "Teardown complete does not authorize future re-use of the same environment identity." + ] +} diff --git a/tools/validate_runtime_sandbox_run.py b/tools/validate_runtime_sandbox_run.py index 2d0c1c6..81c961b 100644 --- a/tools/validate_runtime_sandbox_run.py +++ b/tools/validate_runtime_sandbox_run.py @@ -6,13 +6,17 @@ from typing import Any ROOT = Path(__file__).resolve().parents[1] +SANDBOX = ROOT / "tests" / "fixtures" / "sandbox" VALID_FIXTURES = [ - ROOT / "tests" / "fixtures" / "sandbox" / "runtime-sandbox-run.requested.valid.json", - ROOT / "tests" / "fixtures" / "sandbox" / "runtime-sandbox-run.allocated.valid.json", - ROOT / "tests" / "fixtures" / "sandbox" / "runtime-sandbox-run.failed.valid.json", + SANDBOX / "runtime-sandbox-run.requested.valid.json", + SANDBOX / "runtime-sandbox-run.allocated.valid.json", + SANDBOX / "runtime-sandbox-run.failed.valid.json", + SANDBOX / "runtime-sandbox-run.shared-receipt.valid.json", + SANDBOX / "runtime-sandbox-run.teardown.valid.json", ] INVALID_FIXTURES = [ - ROOT / "tests" / "fixtures" / "sandbox" / "runtime-sandbox-run.allocated.missing-leakcheck.invalid.json", + SANDBOX / "runtime-sandbox-run.allocated.missing-leakcheck.invalid.json", + SANDBOX / "runtime-sandbox-run.teardown.missing-evidence.invalid.json", ] STATUSES = {"runtime_requested", "runtime_allocated", "runtime_failed", "runtime_teardown_complete"} PARITY_LEVELS = {"contract_only", "runtime_observed"} @@ -112,6 +116,17 @@ def validate(data: dict[str, Any]) -> list[str]: problems.append("runtime_failed requires runtime_allocation_failed") if data.get("teardownState") != "teardown_failed": problems.append("runtime_failed teardownState must be teardown_failed") + if status == "runtime_teardown_complete": + if parity != "runtime_observed": + problems.append("runtime_teardown_complete must be runtime_observed") + if not evidence_refs: + problems.append("runtime_teardown_complete requires evidence refs") + if not receipt_refs: + problems.append("runtime_teardown_complete requires receipt refs") + if failure_codes: + problems.append("runtime_teardown_complete must not have failure codes") + if data.get("teardownState") != "teardown_complete": + problems.append("runtime_teardown_complete teardownState must be teardown_complete") return problems