From e1b7b21c04c055eba4f430f98742eea6c69002d2 Mon Sep 17 00:00:00 2001 From: Jin Xiao Date: Mon, 5 Oct 2026 04:42:08 +0800 Subject: [PATCH 1/2] fix(ci): run maintained runtime simulation and interoperability suites --- .github/workflows/ci.yml | 62 ++++++++++ docs/LOCAL_INTEGRATION.md | 30 +++++ docs/MAINTAINING.md | 16 ++- docs/RELEASING.md | 3 +- integration/local/batch_report.py | 5 +- integration/local/ci_report.py | 115 +++++++++++++++++++ integration/local/run.py | 14 +++ integration/local/stream_run.py | 5 +- tools/reference/generate-datamasking-kms.mjs | 12 +- tools/release.py | 6 +- tools/release_automation.py | 2 +- tools/test_release.py | 1 + tools/test_release_automation.py | 19 ++- tools/test_runtime_ci.py | 110 ++++++++++++++++++ 14 files changed, 386 insertions(+), 14 deletions(-) create mode 100644 integration/local/ci_report.py create mode 100644 tools/test_runtime_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3986c2a..7114aca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,3 +66,65 @@ jobs: path: dist/modules/check-*/progress.json if-no-files-found: ignore retention-days: 7 + + runtime: + name: Runtime simulation + needs: test + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + POWERTOOLS_ACCEPTANCE_DIR: ${{ github.workspace }}/dist/runtime-acceptance + GOMAXPROCS: '2' + GOFLAGS: '-p=1' + GOMEMLIMIT: 128MiB + GOGC: '20' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache-dependency-path: '**/go.sum' + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: '22.21.1' + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + python-version: '3.14' + enable-cache: false + - name: Test runtime evidence safeguards + run: uv run --no-project python tools/test_runtime_ci.py + - name: Install pinned TypeScript reference dependencies + run: npm ci --prefix tools/reference + - name: Build both architectures and run Lambda, streaming, and Valkey simulations + run: uv run --no-project python integration/local/run.py --skip-module-checks + - name: Verify Batch logs and spans from this run + run: uv run --no-project python integration/local/batch_report.py + - name: Build KMS interoperability executable + run: go build -trimpath -o dist/local/amd64/kmsinterop ./integration/kmsinterop + env: + GOOS: linux + GOARCH: amd64 + - name: Verify Go ciphertext with the pinned TypeScript provider + working-directory: tools/reference + run: node generate-datamasking-kms.mjs --verify-go ../../dist/local/amd64/kmsinterop + - name: Validate and summarize current-commit runtime evidence + if: always() + run: uv run --no-project python integration/local/ci_report.py + - name: Ensure simulations preserve checked-in acceptance history + run: git diff --exit-code + - name: Upload runtime acceptance and diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: runtime-simulation-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + dist/runtime-acceptance/*.json + dist/runtime-acceptance/*.md + dist/local/*.json + dist/local/*.log + dist/local/streaming/*.json + dist/local/streaming/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/docs/LOCAL_INTEGRATION.md b/docs/LOCAL_INTEGRATION.md index 355d49d..db5f820 100644 --- a/docs/LOCAL_INTEGRATION.md +++ b/docs/LOCAL_INTEGRATION.md @@ -2,6 +2,36 @@ Latest recorded runtime acceptance (2026-10-04, Logger TextMarshaler follow-up): 904/904 Lambda RIE assertions, 95/95 streaming checks and 14/14 saved Batch checks, including six new assertions for empty-string TextMarshaler values/errors. All 31 packaged modules and 28 public consumers passed one complete check; the same flow rebuilt normal and streaming handlers for Linux amd64/arm64 with CGO disabled. Docker executed amd64. Docker Desktop was restarted before validation after an engine HTTP 500; a private temporary container kept the engine active during the complete run and was removed afterward. Runtime metadata records that module checks and builds both executed. See [packaged evidence](MODULE_ACCEPTANCE_LOGGER.json), [acceptance history](LOCAL_VALIDATION.md) and [runtime evidence](LOCAL_ACCEPTANCE.json). No AWS resources were used; exhaustive parity and browser review remain separate gates. +## GitHub Actions + +Go CI runs **Runtime simulation** on every pull request, main push, and manual +dispatch after **Modules and Lambda artifacts** passes. It installs Node.js +22.21.1 and reference dependencies with `npm ci`, then uses +`integration/local/run.py --skip-module-checks`: module tests are reused from +the prerequisite job while normal and streaming fixtures are freshly built for +Linux amd64 and arm64. Docker executes amd64 Lambda RIE, streaming, and the live +TypeScript/Go Valkey bridge. Batch checks consume that same run's logs/spans; +the separate KMS executable emits Go ciphertext for the pinned TypeScript +provider's reverse interoperability checks. All endpoints and credentials are +synthetic; no AWS secrets or cloud resources are required. + +CI sets `POWERTOOLS_ACCEPTANCE_DIR` to the ignored `dist/runtime-acceptance` +directory so Streaming, Batch, and KMS reports preserve the checked-in local +acceptance history. Each report identifies its commit, workflow run, and attempt. +`integration/local/ci_report.py` rejects missing/empty/failed suites, stale +identity, reused builds, missing binaries, Batch artifact hash mismatches, and +cleanup errors. It writes a job summary and SHA-256 hashes for all six binaries. +The `runtime-simulation-SHA-RUN-ATTEMPT` artifact retains reports and diagnostic +logs for seven days, including on failure; build caches and binaries are excluded. + +Both merge protection and release publication must require **Runtime simulation**. +See [maintainer setup](MAINTAINING.md) for rollout. Checked-in Go reference +corpora run in the module job; the full Node fixture regeneration command remains +a deliberate development operation when reference versions or generators change. +Runtime simulation does not establish arm64 execution or live AWS acceptance. + +## Local execution + Run from the repository root with local Go, Python 3.10+ (or `uv`), Node.js with the pinned reference dependencies, and Docker Desktop using Linux containers: ```powershell diff --git a/docs/MAINTAINING.md b/docs/MAINTAINING.md index 2b1cdcd..9c4ad1e 100644 --- a/docs/MAINTAINING.md +++ b/docs/MAINTAINING.md @@ -14,10 +14,10 @@ merge requirements. This guide does not claim those settings are already active. that issue, Summary, and Testing filled in. There is no special owner, `no-issue`, label, or bootstrap bypass in the policy check. 3. Verify the first **PR contribution policy**, **Modules and Lambda artifacts**, - and **Build documentation** runs on that PR's latest revision. Inspect the + **Runtime simulation**, and **Build documentation** runs on that PR's latest revision. Inspect the workflow diff before approving a fork workflow run. 4. Configure the protection below **before merging the bootstrap PR**, selecting - the three checks from their observed successful runs. Do not require a check + the four checks from their observed successful runs. Do not require a check that has never run or was renamed. Confirm the PR's merge box applies the rules. 5. Merge the reviewed bootstrap PR through the protected PR interface. Forms become available once they are on the default branch; verify the New issue @@ -38,8 +38,8 @@ needed for basic PR and status-check protection. only one active maintainer. Do not enable code-owner or last-push approval requirements until another eligible reviewer is available - Require conversation resolution and these exact, stable status-check job names: - **PR contribution policy**, **Modules and Lambda artifacts**, and - **Build documentation**. Select GitHub Actions as the expected source where + **PR contribution policy**, **Modules and Lambda artifacts**, + **Runtime simulation**, and **Build documentation**. Select GitHub Actions as the expected source where available, and require the branch to be up to date before merging - Block force pushes and deletion of `main` - Leave the bypass list empty, including repository administrators. For classic @@ -56,6 +56,14 @@ and [author review limitation](https://docs.github.com/en/pull-requests/how-tos/ ## Triage and safe CI +When introducing **Runtime simulation** to an existing protected repository, +first verify its native PR run and commit-associated artifacts. Then add that +exact job name to the existing main ruleset's required status checks, selecting +GitHub Actions as the expected source. Preserve every existing rule and bypass +setting. Enable the check before merging the CI rollout PR. The publication +tool also requires runtime success on both the preparation PR and its exact +merged main commit; missing, pending, failed, or skipped checks cannot publish. + Use issue comments to record accepted scope and remaining questions. Optional labels are organizational only; no custom label creation, project board, assignment, CLA, or signing service is required to start contributing. If a diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 1e0b7a7..7b783b7 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -75,7 +75,8 @@ recorded source must still be its first parent; if main advances first, rerun preparation against current main and use the newly generated PR. With automatic publication enabled, merging this preparation PR authorizes its -frozen release batch. After both required main checks pass, GoReleaser publishes +frozen release batch. After all three required main checks pass (Modules and Lambda +artifacts, Runtime simulation, and Build documentation), GoReleaser publishes in dependency order and the tracking issue closes after public consumer checks. Ordinary feature/bug PR merges do not publish modules. diff --git a/integration/local/batch_report.py b/integration/local/batch_report.py index 4956845..7012562 100644 --- a/integration/local/batch_report.py +++ b/integration/local/batch_report.py @@ -5,6 +5,8 @@ import json from pathlib import Path +from run import acceptance_path, execution_identity + root = Path(__file__).resolve().parents[2] directory = root / "dist/local" report = json.loads((directory / "report.json").read_text(encoding="utf-8")) @@ -48,7 +50,8 @@ def check(name, value): check("three successful invocation probes", sum("batch" in value["response"] for value in report["invocations"]) == 3) summary = {"source": "existing local Docker artifacts; no additional invocations", "passed": all(item["passed"] for item in checks), "checks": checks, "sha256": {name: hashlib.sha256((directory / name).read_bytes()).hexdigest() for name in ("report.json", "otlp.json", "lambda.log")}} -(root / "docs/BATCH_ACCEPTANCE.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") +summary["execution"] = execution_identity() +acceptance_path("BATCH_ACCEPTANCE.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") failed = [item["name"] for item in checks if not item["passed"]] print(f"Batch composition: {len(checks) - len(failed)}/{len(checks)} checks passed") if failed: diff --git a/integration/local/ci_report.py b/integration/local/ci_report.py new file mode 100644 index 0000000..484e543 --- /dev/null +++ b/integration/local/ci_report.py @@ -0,0 +1,115 @@ +"""Validate fresh runtime evidence and write the GitHub Actions job summary.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[2] + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def summarize(root, identity): + acceptance = root / "dist/runtime-acceptance" + suites = { + "Lambda RIE and Valkey": (root / "dist/local/report.json", "passed"), + "Streaming": (root / "dist/local/streaming/report.json", "passed"), + "Batch logs and spans": (acceptance / "BATCH_ACCEPTANCE.json", "passed"), + "Go to TypeScript KMS": (acceptance / "DATAMASKING_KMS_INTEROP.json", "completed"), + } + summary = {"execution": identity, "runtime_architecture": "amd64", + "binary_architectures": ["amd64", "arm64"], "cgo_enabled": False, + "module_checks": "reused from prerequisite Modules and Lambda artifacts job", + "suites": {}, "binary_sha256": {}, "errors": []} + errors = summary["errors"] + for key in ("source_sha", "run_id", "run_attempt"): + if not identity.get(key): + errors.append(f"Missing execution identity: {key}") + for name, (path, flag) in suites.items(): + try: + report = json.loads(path.read_text(encoding="utf-8")) + checks = report.get("checks", []) + passed = (report.get(flag) is True and bool(checks) + and all(check.get("passed") is True for check in checks)) + if not passed: + errors.append(f"{name}: missing or failed assertions") + if report.get("execution") != identity: + errors.append(f"{name}: evidence belongs to another commit or run attempt") + passed = False + if report.get("cleanup_errors"): + errors.append(f"{name}: resource cleanup failed") + passed = False + if name in ("Lambda RIE and Valkey", "Streaming"): + if (report.get("runtime_architecture") != "amd64" + or report.get("binary_architectures") != ["amd64", "arm64"] + or report.get("cgo_enabled") is not False): + errors.append(f"{name}: unexpected runtime or build configuration") + passed = False + if name == "Lambda RIE and Valkey": + if report.get("builds_executed") is not True or report.get("go_validation_executed") is not False: + errors.append(f"{name}: expected fresh builds and reused module checks") + passed = False + if name == "Batch logs and spans": + for filename in ("report.json", "otlp.json", "lambda.log"): + if report.get("sha256", {}).get(filename) != digest(root / "dist/local" / filename): + errors.append(f"{name}: source artifact changed: {filename}") + passed = False + summary["suites"][name] = {"passed": passed, "assertions": len(checks)} + except (OSError, ValueError, TypeError, AttributeError) as error: + errors.append(f"{name}: {error}") + summary["suites"][name] = {"passed": False, "assertions": 0} + binaries = [f"{arch}/{name}" for arch in ("amd64", "arm64") + for name in ("bootstrap", "stream-bootstrap")] + binaries.extend(("amd64/capture", "amd64/kmsinterop")) + for filename in binaries: + try: + summary["binary_sha256"][filename] = digest(root / "dist/local" / filename) + except OSError as error: + errors.append(f"Missing built binary {filename}: {error}") + summary["passed"] = not errors + return summary + + +def render(summary): + identity = summary["execution"] + lines = ["## Runtime simulation", "", + f"Commit: `{identity.get('source_sha', 'missing')}`; run: " + f"`{identity.get('run_id', 'missing')}`; attempt: `{identity.get('run_attempt', 'missing')}`.", "", + "| Suite | Assertions | Result |", "| --- | ---: | --- |"] + for name, suite in summary["suites"].items(): + lines.append(f"| {name} | {suite['assertions']} | {'Passed' if suite['passed'] else 'Failed'} |") + lines.extend(("", "Module checks ran in the prerequisite job. Both architectures were rebuilt; " + "Docker executed amd64. CGO was disabled. Local synthetic endpoints only; no AWS resources.", "")) + lines.extend(f"- {error}" for error in summary["errors"]) + return "\n".join(lines) + "\n" + + +def main(): + identity = {key: os.environ.get(name, "") for key, name in ( + ("source_sha", "GITHUB_SHA"), ("run_id", "GITHUB_RUN_ID"), ("run_attempt", "GITHUB_RUN_ATTEMPT"))} + summary = summarize(ROOT, identity) + head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, + capture_output=True, text=True).stdout.strip() + if head != identity["source_sha"]: + summary["errors"].append("Checked-out commit does not match GITHUB_SHA") + summary["passed"] = False + directory = ROOT / "dist/runtime-acceptance" + directory.mkdir(parents=True, exist_ok=True) + (directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + markdown = render(summary) + (directory / "summary.md").write_text(markdown, encoding="utf-8") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as output: + output.write(markdown) + print(markdown) + if not summary["passed"]: + raise SystemExit("Runtime acceptance evidence is incomplete or failed") + + +if __name__ == "__main__": + main() diff --git a/integration/local/run.py b/integration/local/run.py index 5e21cd3..74d35e4 100644 --- a/integration/local/run.py +++ b/integration/local/run.py @@ -17,6 +17,19 @@ IMAGE = "public.ecr.aws/lambda/provided@sha256:0b17e5c778aef6ed7f61cbfa5dd416e17540e127846250646fceb712e0e7be5f" +def execution_identity(): + return {key: os.environ[name] for key, name in ( + ("source_sha", "GITHUB_SHA"), ("run_id", "GITHUB_RUN_ID"), + ("run_attempt", "GITHUB_RUN_ATTEMPT"), + ) if os.environ.get(name)} + + +def acceptance_path(name): + directory = Path(os.environ.get("POWERTOOLS_ACCEPTANCE_DIR", ROOT / "docs")) + directory.mkdir(parents=True, exist_ok=True) + return directory / name + + def run(*args, env=None): result = subprocess.run(args, cwd=ROOT, env=env, text=True, encoding="utf-8", capture_output=True) if result.returncode: @@ -72,6 +85,7 @@ def main(): network, capture, function, cache = prefix, prefix + "-capture", prefix + "-lambda", prefix + "-cache" containers = [] report = {"cgo_enabled": False, "image": IMAGE, "runtime_architecture": "amd64", "go_validation_executed": not args.runtime_only and not args.skip_module_checks, "builds_executed": not args.runtime_only, "binary_architectures": ["amd64", "arm64"], "checks": [], "invocations": [], "completed": False} + report["execution"] = execution_identity() def check(name, ok): report["checks"].append({"name": name, "passed": bool(ok)}) diff --git a/integration/local/stream_run.py b/integration/local/stream_run.py index 98e9e76..b731da3 100644 --- a/integration/local/stream_run.py +++ b/integration/local/stream_run.py @@ -8,7 +8,7 @@ import uuid from pathlib import Path -from run import IMAGE, OUT, ROOT, prepare, run +from run import IMAGE, OUT, ROOT, acceptance_path, execution_identity, prepare, run def main(): @@ -23,6 +23,7 @@ def main(): network, capture, function = prefix, prefix + '-capture', prefix + '-lambda' containers = [] report = {'scope': 'real aws-lambda-go SDK against a local Runtime API fixture; not AWS service acceptance', 'completed': False, 'cgo_enabled': False, 'runtime_architecture': 'amd64', 'binary_architectures': ['amd64', 'arm64'], 'image': IMAGE, 'checks': [], 'invocations': [], 'cleanup_errors': []} + report['execution'] = execution_identity() def check(name, passed): report['checks'].append({'name': name, 'passed': bool(passed)}) @@ -128,7 +129,7 @@ def request(path, value=None): if not report['passed']: raise RuntimeError(f"Stream acceptance failures: {[check['name'] for check in report['checks'] if not check['passed']]}; cleanup: {report['cleanup_errors']}") report['assertion_count'] = len(report['checks']) - (ROOT / 'docs/STREAMING_ACCEPTANCE.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8', newline='\n') + acceptance_path('STREAMING_ACCEPTANCE.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8', newline='\n') print(f"Streaming: {len(report['checks'])}/{len(report['checks'])} checks passed; real Go Lambda SDK, local Runtime API fixture") diff --git a/tools/reference/generate-datamasking-kms.mjs b/tools/reference/generate-datamasking-kms.mjs index 0a85156..4008cc9 100644 --- a/tools/reference/generate-datamasking-kms.mjs +++ b/tools/reference/generate-datamasking-kms.mjs @@ -3,7 +3,7 @@ import { randomBytes } from 'node:crypto'; import { createRequire } from 'node:module'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; -import { resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { isDeepStrictEqual } from 'node:util'; const key = 'arn:aws:kms:ap-east-1:111122223333:key/11111111-1111-4111-8111-111111111111'; @@ -62,7 +62,15 @@ try { checks.push({ name: item.name + ': context mismatch', passed: wrong.error?.name === 'DataMaskingEncryptionError' && wrong.error.message === "Encryption context mismatch for key 'missing'" }); } const report = { scope: 'Actual AWS Encryption SDK messages through local KMS fixtures; no real KMS/cache acceptance', typescript: '2.35.0', clientNode: '5.0.2', goSDK: '0.4.0', completed: checks.every(check => check.passed), requests, checks }; - writeFileSync('../../docs/DATAMASKING_KMS_INTEROP.json', JSON.stringify(report, null, 2) + '\n'); + report.execution = Object.fromEntries([ + ['source_sha', process.env.GITHUB_SHA], ['run_id', process.env.GITHUB_RUN_ID], + ['run_attempt', process.env.GITHUB_RUN_ATTEMPT], + ].filter(([, value]) => value)); + const reportPath = process.env.POWERTOOLS_ACCEPTANCE_DIR + ? resolve(process.env.POWERTOOLS_ACCEPTANCE_DIR, 'DATAMASKING_KMS_INTEROP.json') + : '../../docs/DATAMASKING_KMS_INTEROP.json'; + mkdirSync(dirname(reportPath), { recursive: true }); + writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n'); if (!report.completed) throw new Error('Go/TypeScript interoperability failed'); console.log(`${checks.length}/${checks.length} Go-to-TypeScript checks passed`); } else { diff --git a/tools/release.py b/tools/release.py index c976999..78e624e 100644 --- a/tools/release.py +++ b/tools/release.py @@ -19,6 +19,8 @@ ROOT = Path(__file__).resolve().parents[1] REPOSITORY = "rambow-cloud/powertools-lambda-go" GORELEASER_VERSION = "2.18.2" +MAIN_CHECKS = ("Modules and Lambda artifacts", "Runtime simulation", "Build documentation") +PR_CHECKS = ("PR contribution policy", *MAIN_CHECKS) SHA = re.compile(r"[0-9a-f]{40}") NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,79}") VERSION = re.compile(r"v(0|1)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?") @@ -377,8 +379,8 @@ def preflight(args, api): changed = set(git("diff", "--name-only", plan["source_sha"], args.sha).splitlines()) if not changed <= allowed or f"releases/{args.plan}.json" not in changed: raise ValueError("Preparation PR must only change its release plan and module/dependency/license metadata; ship code changes in earlier PRs.") - check_runs(api, pr["head"]["sha"], ["PR contribution policy", "Modules and Lambda artifacts", "Build documentation"]) - check_runs(api, args.sha, ["Modules and Lambda artifacts", "Build documentation"]) + check_runs(api, pr["head"]["sha"], PR_CHECKS) + check_runs(api, args.sha, MAIN_CHECKS) requirements = {} env = os.environ.copy() env.update(CGO_ENABLED="0", GOWORK="off") diff --git a/tools/release_automation.py b/tools/release_automation.py index 6e3f138..406f2fe 100644 --- a/tools/release_automation.py +++ b/tools/release_automation.py @@ -62,7 +62,7 @@ def automatic_preparation(event, api): print("No publication: the release tracking issue is already closed.") return None try: - release.check_runs(api, sha, ["Modules and Lambda artifacts", "Build documentation"]) + release.check_runs(api, sha, release.MAIN_CHECKS) except ValueError: print("No publication yet: required main checks have not all passed. Their next completion event will re-evaluate this batch.") return None diff --git a/tools/test_release.py b/tools/test_release.py index 50ade9e..8c3b9a0 100644 --- a/tools/test_release.py +++ b/tools/test_release.py @@ -437,6 +437,7 @@ def git(*args): if scenario == "valid": self.assertEqual(release.preflight(args, api)[3], ["logger"]) self.assertEqual([call.args[1] for call in checks.call_args_list], [C, B]) + self.assertTrue(all("Runtime simulation" in call.args[2] for call in checks.call_args_list)) else: with self.assertRaises(ValueError): release.preflight(args, api) diff --git a/tools/test_release_automation.py b/tools/test_release_automation.py index 3cc43fc..4ffac26 100644 --- a/tools/test_release_automation.py +++ b/tools/test_release_automation.py @@ -38,6 +38,7 @@ def __init__(self): self.issue_state = "open" self.native = None self.pr_head = None + self.runtime_check = {"status": "completed", "conclusion": "success"} def request(self, path, **kwargs): return {"login": "maintainer"} @@ -64,7 +65,10 @@ def repo(self, path, **kwargs): sha = path.split("head_sha=", 1)[1].split("&", 1)[0] return {"workflow_runs": self.native if self.native is not None else native_runs(sha)} if path.startswith("commits/") and "/check-runs?" in path: - return {"check_runs": [{"id": number, "name": name, "app": {"slug": "github-actions"}, "status": "completed", "conclusion": "success"} for number, name in enumerate(("PR contribution policy", "Modules and Lambda artifacts", "Build documentation"))]} + checks = [{"id": number, "name": name, "app": {"slug": "github-actions"}, "status": "completed", "conclusion": "success"} for number, name in enumerate(("PR contribution policy", "Modules and Lambda artifacts", "Build documentation"))] + if self.runtime_check is not None: + checks.append({"id": 4, "name": "Runtime simulation", "app": {"slug": "github-actions"}, **self.runtime_check}) + return {"check_runs": checks} if path.startswith("git/ref/tags/"): sha = self.tag_shas.get(path.removeprefix("git/ref/tags/")) return {"object": {"type": "commit", "sha": sha}} if sha else None @@ -379,6 +383,19 @@ def repo(path, **kwargs): class CompletionTests(unittest.TestCase): + def test_publication_waits_for_successful_runtime_simulation(self): + with fixture() as (root, api, source): + event = self.merged(root, api, source) + for check in (None, {"status": "in_progress", "conclusion": None}, + {"status": "completed", "conclusion": "failure"}, + {"status": "completed", "conclusion": "skipped"}, + {"status": "completed", "conclusion": "success"}): + with self.subTest(check=check): + api.runtime_check = check + result = completion.automatic_preparation(event, api) + self.assertEqual(result is not None, check is not None and check["conclusion"] == "success") + self.assertEqual(api.writes, []) + def test_legacy_schema_one_plan_can_still_be_resolved_for_recovery(self): with fixture() as (root, api, source): event = self.merged(root, api, source) diff --git a/tools/test_runtime_ci.py b/tools/test_runtime_ci.py new file mode 100644 index 0000000..424348c --- /dev/null +++ b/tools/test_runtime_ci.py @@ -0,0 +1,110 @@ +"""Offline regressions for runtime CI evidence; no Docker or AWS calls.""" + +from copy import deepcopy +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "integration/local")) +import ci_report +import run as runtime + + +class RuntimeEvidenceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.identity = {"source_sha": "a" * 40, "run_id": "101", "run_attempt": "1"} + self.rie = self.root / "dist/local/report.json" + self.stream = self.root / "dist/local/streaming/report.json" + self.batch = self.root / "dist/runtime-acceptance/BATCH_ACCEPTANCE.json" + self.kms = self.root / "dist/runtime-acceptance/DATAMASKING_KMS_INTEROP.json" + report = {"execution": self.identity, "passed": True, "completed": True, + "checks": [{"passed": True}], "runtime_architecture": "amd64", + "binary_architectures": ["amd64", "arm64"], "cgo_enabled": False, + "builds_executed": True, "go_validation_executed": False, "cleanup_errors": []} + for path in (self.rie, self.stream, self.batch, self.kms): + self.write(path, deepcopy(report)) + for arch in ("amd64", "arm64"): + for name in ("bootstrap", "stream-bootstrap"): + (self.root / f"dist/local/{arch}").mkdir(parents=True, exist_ok=True) + (self.root / f"dist/local/{arch}/{name}").write_bytes(b"synthetic executable") + for name in ("capture", "kmsinterop"): + (self.root / f"dist/local/amd64/{name}").write_bytes(b"synthetic executable") + for name in ("otlp.json", "lambda.log"): + (self.root / "dist/local" / name).write_text("synthetic evidence", encoding="utf-8") + self.bind_batch() + + def write(self, path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data), encoding="utf-8") + + def update(self, path, **changes): + data = json.loads(path.read_text(encoding="utf-8")) + data.update(changes) + self.write(path, data) + + def bind_batch(self): + self.update(self.batch, sha256={name: ci_report.digest(self.root / "dist/local" / name) + for name in ("report.json", "otlp.json", "lambda.log")}) + + def test_current_complete_evidence_passes_and_records_binaries(self): + summary = ci_report.summarize(self.root, self.identity) + self.assertTrue(summary["passed"], summary["errors"]) + self.assertEqual(len(summary["binary_sha256"]), 6) + self.assertEqual(sum(item["assertions"] for item in summary["suites"].values()), 4) + self.assertIn(self.identity["source_sha"], ci_report.render(summary)) + + def test_missing_report_does_not_fall_back_to_documented_history(self): + self.kms.unlink() + self.write(self.root / "docs/DATAMASKING_KMS_INTEROP.json", {"completed": True}) + summary = ci_report.summarize(self.root, self.identity) + self.assertFalse(summary["passed"]) + self.assertFalse(summary["suites"]["Go to TypeScript KMS"]["passed"]) + + def test_other_commits_runs_and_attempts_are_rejected(self): + for key, value in (("source_sha", "b" * 40), ("run_id", "102"), ("run_attempt", "2")): + with self.subTest(key=key): + self.update(self.kms, execution={**self.identity, key: value}) + self.assertFalse(ci_report.summarize(self.root, self.identity)["passed"]) + + def test_failed_and_empty_checks_cannot_hide_behind_success_flag(self): + for checks in ([], [{"passed": False}], [{"passed": "true"}]): + with self.subTest(checks=checks): + self.update(self.kms, checks=checks) + self.assertFalse(ci_report.summarize(self.root, self.identity)["passed"]) + + def test_cleanup_failure_is_fatal_even_when_assertions_pass(self): + self.update(self.stream, cleanup_errors=["synthetic removal failure"]) + self.assertFalse(ci_report.summarize(self.root, self.identity)["passed"]) + + def test_reused_binaries_cannot_be_claimed_as_fresh_builds(self): + self.update(self.rie, builds_executed=False) + self.bind_batch() + self.assertFalse(ci_report.summarize(self.root, self.identity)["passed"]) + + def test_batch_evidence_must_match_current_logs(self): + (self.root / "dist/local/lambda.log").write_text("changed evidence", encoding="utf-8") + self.assertFalse(ci_report.summarize(self.root, self.identity)["passed"]) + + def test_missing_arm64_build_is_fatal(self): + (self.root / "dist/local/arm64/stream-bootstrap").unlink() + self.assertFalse(ci_report.summarize(self.root, self.identity)["passed"]) + + def test_missing_ci_identity_cannot_validate_local_history(self): + self.assertFalse(ci_report.summarize(self.root, {})["passed"]) + + def test_acceptance_output_redirect_preserves_local_default(self): + with patch.object(runtime, "ROOT", self.root), patch.dict(os.environ, {}, clear=True): + self.assertEqual(runtime.acceptance_path("report.json"), self.root / "docs/report.json") + with patch.dict(os.environ, {"POWERTOOLS_ACCEPTANCE_DIR": str(self.root / "dist/evidence")}): + self.assertEqual(runtime.acceptance_path("report.json"), self.root / "dist/evidence/report.json") + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 501444c13656eec752aaf40b7a888005e20b23d4 Mon Sep 17 00:00:00 2001 From: Jin Xiao Date: Mon, 5 Oct 2026 04:58:18 +0800 Subject: [PATCH 2/2] docs: record hosted runtime CI acceptance and required gate --- docs/CHECKLIST.md | 1 + docs/CI_RUNTIME_ACCEPTANCE.json | 46 +++++++++++++++++++++++++++++++++ docs/LOCAL_INTEGRATION.md | 9 +++++++ 3 files changed, 56 insertions(+) create mode 100644 docs/CI_RUNTIME_ACCEPTANCE.json diff --git a/docs/CHECKLIST.md b/docs/CHECKLIST.md index 7de5dfa..de4f2b1 100644 --- a/docs/CHECKLIST.md +++ b/docs/CHECKLIST.md @@ -278,3 +278,4 @@ Race-detector execution is intentionally excluded because CGO must remain disabl - [x] SIGNER-BODY-HANDOFF: Preserve asynchronous base-transport body ownership for custom signers returning the input request or a clone sharing its body. Four deterministic success/error regressions reproduced premature closure before the fix; packaged Signer tests, vet, tidy verification and its independent consumer build pass with CGO disabled. Distinct/empty bodies and nil signing results retain cleanup coverage. See issue #25 and [Signer ownership](SIGNER.md) (2026-10-05). - [x] MODULE-REVIEW-20261005: Review the main entry points and critical state/ownership/error/decoding paths of the 26 maintained public modules excluding Logger. Twenty-five native Windows packaged checks and standalone consumers passed with CGO disabled; the exact-baseline hosted Linux run passed all 31 repository modules, including KMS and frozen X-Ray. Register issues #25–#28 before production changes; merge the verified Signer fix before starting the shared decoding fix. Local Docker runtime acceptance was unavailable. See [the review scope and evidence](MODULE_REVIEW_2026_10_05.md). - [x] UTF8-DECODING: Reuse Commons.DecodeUTF8 in Parser, Parameters and JMESPath while preserving each reference path's BOM and gzip fallback rules. Generate 56 expected cases from the pinned TypeScript 2.35.0 packages: 20 Parser, 16 Parameters and 20 JMESPath cases. New regressions failed before the fix; all three affected packaged modules passed tests, vet, tidy and standalone consumer builds afterward, with CGO disabled and GOWORK=off. Parameters additionally verifies Auto transforms and untransformed raw bytes. See issues #26–#28 and [the review report](MODULE_REVIEW_2026_10_05.md) (2026-10-05). +- [x] CI-RUNTIME: Add native PR/main runtime simulation after packaged checks, fresh amd64/arm64 integration builds, Docker amd64 RIE/streaming/Valkey execution, saved Batch checks and Go-to-TypeScript KMS interoperability. PR #32's Go CI run 37233122495 passed all 31 modules / 28 consumers and 904 + 95 + 14 + 108 runtime assertions with CGO disabled, no cleanup errors, and preserved checked-in acceptance history. Ten offline evidence safeguards, 37 release and 22 automation regressions, actionlint and licenses pass. Require runtime success on preparation PR/main release gates and add the observed GitHub Actions check to the existing active main ruleset, preserving the other rules. No cloud AWS or arm64 execution is claimed. See issue #31 and [the hosted acceptance summary](CI_RUNTIME_ACCEPTANCE.json) (2026-10-05). diff --git a/docs/CI_RUNTIME_ACCEPTANCE.json b/docs/CI_RUNTIME_ACCEPTANCE.json new file mode 100644 index 0000000..5f401a7 --- /dev/null +++ b/docs/CI_RUNTIME_ACCEPTANCE.json @@ -0,0 +1,46 @@ +{ + "execution": { + "source_sha": "bf024a84d1317262d80da2f94d664d6a5d048a91", + "run_id": "37233122495", + "run_attempt": "1" + }, + "runtime_architecture": "amd64", + "binary_architectures": [ + "amd64", + "arm64" + ], + "cgo_enabled": false, + "module_checks": "reused from prerequisite Modules and Lambda artifacts job", + "suites": { + "Lambda RIE and Valkey": { + "passed": true, + "assertions": 904 + }, + "Streaming": { + "passed": true, + "assertions": 95 + }, + "Batch logs and spans": { + "passed": true, + "assertions": 14 + }, + "Go to TypeScript KMS": { + "passed": true, + "assertions": 108 + } + }, + "binary_sha256": { + "amd64/bootstrap": "165758ad2d24e7e4288cbf34ec173ea8ba7da4a033634c319041836af9893c90", + "amd64/stream-bootstrap": "ab01acbad6032dcdeeb7d6932fc95482dfd66fa0202c3c9eaf0841be451ec53a", + "arm64/bootstrap": "752fbe75ae5ad7b0427a458f158b07cb78000ec446943bd65ab6e5a4fef78eb3", + "arm64/stream-bootstrap": "6e7a7f5f19ea98a64dc21c89821ecd2bb50b6f55bc708aab1abf2a4bfc4bcc01", + "amd64/capture": "474b5493d9b79476cfa66396c0935a7447e3a68937821981fe18bf88cd98d50d", + "amd64/kmsinterop": "eb184304aa6f626ad6971a4c34ae88220d01f7d74c5b28c7d6a773fd9c117ac4" + }, + "errors": [], + "passed": true, + "recorded_at": "2026-10-05", + "workflow_url": "https://github.com/rambow-cloud/powertools-lambda-go/actions/runs/37233122495", + "pull_request": 32, + "pull_request_head_sha": "e1b7b21c04c055eba4f430f98742eea6c69002d2" +} diff --git a/docs/LOCAL_INTEGRATION.md b/docs/LOCAL_INTEGRATION.md index db5f820..47e2103 100644 --- a/docs/LOCAL_INTEGRATION.md +++ b/docs/LOCAL_INTEGRATION.md @@ -4,6 +4,15 @@ Latest recorded runtime acceptance (2026-10-04, Logger TextMarshaler follow-up): ## GitHub Actions +First hosted runtime acceptance (2026-10-05): PR #32's native +[Go CI run](https://github.com/rambow-cloud/powertools-lambda-go/actions/runs/37233122495) +passed all 31 packaged modules, 28 public consumers, 904 RIE/Valkey assertions, +95 streaming checks, 14 Batch checks, and 108 KMS reverse-interop checks. +The runtime rebuilt both architectures, executed amd64, preserved checked-in +reports, and cleaned up without errors. The [acceptance summary](CI_RUNTIME_ACCEPTANCE.json) +records the tested PR merge SHA, run/attempt, suite counts, and six binary hashes; +it is distinct from the historical local acceptance above. + Go CI runs **Runtime simulation** on every pull request, main push, and manual dispatch after **Modules and Lambda artifacts** passes. It installs Node.js 22.21.1 and reference dependencies with `npm ci`, then uses