From 1c5922e57391bfc8d5e601a216afc5537bff18a1 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 13 Jul 2026 23:30:27 +0200 Subject: [PATCH 01/16] fix(deploy): refresh Flux GHCR auth before reconcile --- .github/actions/deploy-prod/action.yml | 20 +- .github/workflows/ci.yaml | 11 + .github/workflows/dr-rebuild.yaml | 6 + AGENTS.md | 19 +- docs/dr/runbook.md | 57 +++++ scripts/refresh-flux-ghcr-auth.sh | 64 ++++++ scripts/tests/test_refresh_flux_ghcr_auth.py | 220 +++++++++++++++++++ 7 files changed, 393 insertions(+), 4 deletions(-) create mode 100755 scripts/refresh-flux-ghcr-auth.sh create mode 100644 scripts/tests/test_refresh_flux_ghcr_auth.py diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index a70bfd29e..d6f64739c 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -19,7 +19,7 @@ inputs: description: talosconfig for the prod cluster (secrets.TALOS_CONFIG) used by `ksail cluster update`. required: true ghcr-token: - description: GHCR PAT with write:packages (secrets.GHCR_TOKEN) for OCI push/pull and cosign auth. + description: GHCR PAT with write:packages (secrets.GHCR_TOKEN) for OCI push, signing, and Talos/KSail compatibility; Flux pulls use the Git/SOPS credential. required: true hcloud-token: description: Hetzner Cloud API token (secrets.HCLOUD_TOKEN) used by the KSail Hetzner provider and CCM/CSI. @@ -284,6 +284,14 @@ runs: push-to-registry: true create-storage-record: false + - name: 🔑 Refresh Flux GHCR pull credential + id: refresh_flux_ghcr_auth + # Git/SOPS is authoritative for pulls. Refresh the KSail-managed root + # Secret before reconciliation so a rotation cannot deadlock Flux on the + # stale credential it needs to fetch the artifact containing the new one. + shell: bash + run: ./scripts/refresh-flux-ghcr-auth.sh + - name: 🔁 Trigger Flux reconciliation id: reconcile shell: bash @@ -321,3 +329,13 @@ runs: # whole machine config ('private key is invalid: wrong key "" length: 0'). WG_SERVER_PRIVATE_KEY: ${{ inputs.wg-server-private-key }} run: ksail --config ksail.prod.yaml cluster update + + - name: 🔑 Reassert Flux GHCR pull credential + id: reassert_flux_ghcr_auth + # cluster update can rewrite KSail's root Secret from GHCR_TOKEN when an + # unrelated configuration change triggers setup. Reassert Git/SOPS even + # when that update fails part-way, provided the source was already proven + # valid by the first refresh. + if: always() && steps.refresh_flux_ghcr_auth.outcome == 'success' + shell: bash + run: ./scripts/refresh-flux-ghcr-auth.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 749ac42dd..415f3df74 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,6 +41,10 @@ jobs: - 'talos/**' - 'scripts/validate-naming.py' - 'scripts/validate-embedded-json.py' + - 'scripts/refresh-flux-ghcr-auth.sh' + - 'scripts/tests/test_refresh_flux_ghcr_auth.py' + - '.github/actions/deploy-prod/**' + - '.github/workflows/dr-rebuild.yaml' - '.github/workflows/ci.yaml' talos: - 'talos/**' @@ -65,6 +69,13 @@ jobs: with: persist-credentials: false + - name: 🔐 Validate Flux GHCR credential bridge + # Pure local tests with fake ksail/docker/kubectl commands: no SOPS key, + # registry credential, cluster access, or network is used. + run: | + bash -n scripts/refresh-flux-ghcr-auth.sh + python3 -m unittest scripts.tests.test_refresh_flux_ghcr_auth + - name: 🧩 Validate embedded JSON blobs # ConfigMaps that embed whole JSON documents as block scalars (e.g. the # Headlamp Kubescape exceptions ConfigMap's exceptionPolicies) are opaque diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index aa83f06dd..60337fff5 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -116,6 +116,12 @@ jobs: GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + - name: 🔑 Refresh Flux GHCR pull credential + # cluster create bootstraps with GHCR_TOKEN for KSail compatibility, + # then this bridge makes the Git/SOPS pull credential authoritative + # before Flux performs its first artifact fetch. + run: ./scripts/refresh-flux-ghcr-auth.sh + - name: 🔁 Trigger Flux reconciliation run: ksail --config ksail.prod.yaml workload reconcile env: diff --git a/AGENTS.md b/AGENTS.md index 62cf1ee5c..70f14f0d1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -145,7 +145,8 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 3. `ksail.prod.yaml` has `kustomizationFile: clusters/prod`, so KSail/Flux use `k8s/clusters/prod/kustomization.yaml` as the entry point — no root `k8s/kustomization.yaml` or file rewriting is needed. 4. `ksail --config ksail.prod.yaml cluster create` (first run) or `cluster update` (subsequent runs) provisions / reconciles the Hetzner servers, Talos, CCM, and CSI. 5. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. -6. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. +6. `scripts/refresh-flux-ghcr-auth.sh` decrypts only the Git/SOPS pull credential, proves it can read the artifact, and refreshes the KSail-managed root Flux Secret before reconciliation. +7. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. **Key differences from local:** @@ -163,14 +164,26 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o - **`ci.yaml`** — runs on `pull_request` (static manifest validation + Kubescape scan, no cluster) and `merge_group` (deploys prod via the Hetzner provider). Concurrency is shared with `cd.yaml` so a manual deploy and a merge-queue deploy can never run against the prod cluster at the same time. - **`cd.yaml`** — runs on `workflow_dispatch` (manual). Deploys to the production Hetzner cluster using `ksail --config ksail.prod.yaml`. Covers direct pushes to `main`, which bypass the merge queue and so are not deployed by `ci.yaml`. -- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (push → cosign-sign → attest SBOM + SLSA provenance → Flux reconcile → Talos `cluster update`), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. +- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (push → cosign-sign → attest SBOM + SLSA provenance → refresh root GHCR auth → Flux reconcile → Talos `cluster update` → reassert root GHCR auth), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. **Required GitHub Secrets:** -- `GHCR_TOKEN` — long-lived PAT (owner: `devantler`) with `write:packages` scope, used for GHCR push/pull authentication. +- `GHCR_TOKEN` — long-lived PAT (owner: `devantler`) with `write:packages` scope, used for OCI push/signing and retained for KSail/Talos compatibility. It is **not** the authoritative Flux pull credential. - `SOPS_AGE_KEY` — Age private key for SOPS secret decryption. - `HCLOUD_TOKEN` — Hetzner Cloud API token (read/write), used by the KSail Hetzner provider and by the Hetzner CCM / CSI at runtime. +The authoritative **Flux and tenant** GHCR pull credential is +`stringData.ghcr_dockerconfigjson` in +`k8s/bases/bootstrap/secret.enc.yaml`. The deploy bridge refreshes +`flux-system/ksail-registry-credentials` from that value before Flux must fetch +the artifact and reasserts it after `cluster update` in case KSail rewrites its +managed Secret. Flux then applies the same SOPS value to `variables-base`; the +`seed-ghcr` PushSecret fans it into OpenBao and tenant ExternalSecrets materialise +their `ghcr-auth` copies. A direct credential commit to `main` still needs a +manual `CD` workflow dispatch because direct pushes bypass the merge-queue deploy. +Talos node registry auth still derives from `GHCR_TOKEN`; consolidating that +remaining pull path stays tracked by #2613 and the KSail credential work. + **Required GitHub Variables:** none. ## Working with Secrets diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index eb28e6ec5..ccca2f9f8 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -164,6 +164,7 @@ ksail --config ksail.prod.yaml cluster create # 3. Bootstrap Flux from this repo ksail --config ksail.prod.yaml workload push # packages -> GHCR +./scripts/refresh-flux-ghcr-auth.sh # Git/SOPS -> root Flux auth ksail --config ksail.prod.yaml workload reconcile # Flux pulls and applies # 4. Wait for Flux to settle @@ -485,6 +486,62 @@ gh secret set TALOS_CONFIG --env prod --repo devantler-tech/platform < ~/.talos/ > step in the deploy workflows is the local guard against that behaviour until > those land. +## Scenario 10 — Flux/tenant GHCR pull credential rotation or denial + +The authoritative Flux and tenant pull credential is the SOPS-encrypted +`stringData.ghcr_dockerconfigjson` value in +`k8s/bases/bootstrap/secret.enc.yaml`. It is deliberately separate from the +GitHub `prod` environment's `GHCR_TOKEN`, which pushes/signs the artifact and is +still passed to KSail/Talos for compatibility. + +Talos node registry auth still derives from `GHCR_TOKEN`; this recovery path +does not rotate existing node machine configuration. That remaining +single-source consolidation stays tracked by #2613. + +The production deploy closes the bootstrap loop in this order: + +1. Push and sign the new artifact with `GHCR_TOKEN`. +2. Decrypt only `ghcr_dockerconfigjson`, verify it can pull the production + artifact, and patch `flux-system/ksail-registry-credentials`. +3. Reconcile Flux. The bootstrap Kustomization applies the same value to + `variables-base`; `seed-ghcr` pushes it to OpenBao and tenant ExternalSecrets + fan it out as `ghcr-auth`. +4. Reassert the root Secret after `cluster update`, because KSail can rewrite its + managed Secret when another cluster setting changes. + +For a normal rotation, update the encrypted value through a PR and let the merge +queue deploy it. If the encrypted file was pushed directly to `main`, manually +dispatch the `CD` workflow: direct pushes do not run the production deploy. + +For an already-denied root source, run the same secret-safe bridge from a clean +`main` checkout with the production Age key and `admin@prod` kube context, then +reconcile the source: + +```bash +./scripts/refresh-flux-ghcr-auth.sh +flux reconcile source oci flux-system -n flux-system --context admin@prod +flux reconcile kustomization flux-system -n flux-system --context admin@prod +``` + +If tenant sources still show `DENIED`, force the existing fan-out after +`variables-base` has converged: + +```bash +stamp=$(date +%s) +kubectl --context admin@prod -n flux-system annotate pushsecret seed-ghcr \ + force-sync="$stamp" --overwrite +for namespace in wedding-app ascoachingogvaner kyverno; do + kubectl --context admin@prod -n "$namespace" annotate externalsecret ghcr-auth \ + force-sync="$stamp" --overwrite +done +flux get sources oci -A --context admin@prod +``` + +Do not replace this bridge with only a root ExternalSecret: Flux needs valid root +auth to fetch the artifact that installs/updates External Secrets, while OpenBao +receives this rotated value only through a downstream PushSecret. That circular +dependency cannot recover a stale root credential by itself. + --- ## Related documents diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh new file mode 100755 index 000000000..9c127d101 --- /dev/null +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Refresh the KSail-managed root Flux pull Secret from the Git/SOPS source. +# +# Flux cannot fetch the artifact containing a rotated credential while its +# bootstrap Secret is stale. Keep this bridge outside Flux so a deployment can +# repair that bootstrap edge before asking Flux to reconcile. + +set -euo pipefail + +readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" +readonly IMAGE="${FLUX_GHCR_IMAGE:-ghcr.io/devantler-tech/platform/manifests:latest}" +readonly KUBE_CONTEXT="${KUBE_CONTEXT:-admin@prod}" + +work_dir="$(mktemp -d)" +trap 'rm -rf "${work_dir}"' EXIT +chmod 700 "${work_dir}" +umask 077 + +docker_config="${work_dir}/config.json" +patch_file="${work_dir}/patch.json" + +# KSail embeds SOPS, so the deploy uses the same pinned toolchain as workload +# reconciliation. Decrypt only the Docker config scalar and never emit it to +# stdout or place its plaintext/base64 representation in an argument. +ksail workload cipher decrypt \ + "${SECRET_FILE}" \ + --extract '["stringData"]["ghcr_dockerconfigjson"]' \ + --output "${docker_config}" \ + >/dev/null +chmod 600 "${docker_config}" + +if ! jq -e ' + (.auths["ghcr.io"] // {}) + | (.username | type == "string" and length > 0) + and (.password | type == "string" and length > 0) +' "${docker_config}" >/dev/null; then + echo "::error::The SOPS GHCR pull credential is not a valid Docker config with non-empty ghcr.io username/password fields." + exit 1 +fi + +# Prove the decrypted credential can actually pull the production artifact +# before mutating the cluster. DOCKER_CONFIG isolates it from the workflow's +# separate push credential in ~/.docker/config.json. +if ! DOCKER_CONFIG="${work_dir}" docker buildx imagetools inspect "${IMAGE}" >/dev/null; then + echo "::error::The SOPS GHCR pull credential cannot read ${IMAGE}; the root Flux Secret was not changed." + exit 1 +fi + +# Merge only the Secret data field so KSail's ownership label and any unrelated +# metadata survive. The sensitive payload stays in the pipe/temp file and never +# appears in argv or logs. +base64 < "${docker_config}" \ + | tr -d '\r\n' \ + | jq -Rs '{data: {".dockerconfigjson": .}}' \ + > "${patch_file}" + +kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + patch secret ksail-registry-credentials \ + --type=merge \ + --patch-file="${patch_file}" + +echo "✅ Refreshed the root Flux GHCR pull credential from Git/SOPS." diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py new file mode 100644 index 000000000..3aff6e5e7 --- /dev/null +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -0,0 +1,220 @@ +"""Regression tests for the production Flux GHCR credential bridge.""" + +from __future__ import annotations + +import base64 +import json +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +HELPER = ROOT / "scripts" / "refresh-flux-ghcr-auth.sh" +ACTION = ROOT / ".github" / "actions" / "deploy-prod" / "action.yml" +DR_REBUILD = ROOT / ".github" / "workflows" / "dr-rebuild.yaml" + + +class RefreshFluxGhcrAuthTests(unittest.TestCase): + """Exercise the helper with fake external commands and no real secrets.""" + + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + self.workspace = Path(self.temp_dir.name) + self.bin_dir = self.workspace / "bin" + self.bin_dir.mkdir() + self.decrypted_config = self.workspace / "decrypted-config.json" + self.patch_capture = self.workspace / "patch.json" + self.kubectl_called = self.workspace / "kubectl-called" + self.output_path_log = self.workspace / "ksail-output-path" + + self._write_executable( + "ksail", + """ + #!/usr/bin/env bash + set -euo pipefail + arguments=" $* " + secret_file='k8s/bases/bootstrap/secret.enc.yaml' + selector='["stringData"]["ghcr_dockerconfigjson"]' + [[ "$arguments" == *" workload cipher decrypt $secret_file "* ]] + [[ "$arguments" == *" --extract $selector "* ]] + output="" + while (($#)); do + case "$1" in + --output) + output="$2" + shift 2 + ;; + *) shift ;; + esac + done + test -n "$output" + printf '%s' "$output" > "$KSAIL_OUTPUT_PATH_LOG" + cp "$FAKE_DECRYPTED_CONFIG" "$output" + """, + ) + self._write_executable( + "docker", + """ + #!/usr/bin/env bash + set -euo pipefail + test "$1" = buildx + test "$2" = imagetools + test "$3" = inspect + test "$4" = ghcr.io/devantler-tech/platform/manifests:latest + test -f "$DOCKER_CONFIG/config.json" + if [[ "${FAKE_DOCKER_FAIL:-false}" == true ]]; then + echo 'registry pull preflight denied' >&2 + exit 42 + fi + """, + ) + self._write_executable( + "kubectl", + """ + #!/usr/bin/env bash + set -euo pipefail + arguments=" $* " + [[ "$arguments" == *" --context admin@prod "* ]] + [[ "$arguments" == *" --namespace flux-system "* ]] + [[ "$arguments" == *" patch secret ksail-registry-credentials "* ]] + [[ "$arguments" == *" --type=merge "* ]] + patch_file="" + for argument in "$@"; do + case "$argument" in + --patch-file=*) patch_file="${argument#*=}" ;; + esac + done + test -n "$patch_file" + touch "$KUBECTL_CALLED" + cp "$patch_file" "$PATCH_CAPTURE" + if [[ "${FAKE_KUBECTL_FAIL:-false}" == true ]]; then + echo 'cluster patch failed' >&2 + exit 43 + fi + echo 'secret/ksail-registry-credentials patched' + """, + ) + + def _write_executable(self, name: str, body: str) -> None: + path = self.bin_dir / name + path.write_text(textwrap.dedent(body).lstrip(), encoding="utf-8") + path.chmod(0o755) + + def _run_helper( + self, config: object, **environment_overrides: str + ) -> subprocess.CompletedProcess[str]: + self.decrypted_config.write_text(json.dumps(config), encoding="utf-8") + for marker in (self.patch_capture, self.kubectl_called, self.output_path_log): + marker.unlink(missing_ok=True) + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.bin_dir}:{environment['PATH']}", + "FAKE_DECRYPTED_CONFIG": str(self.decrypted_config), + "PATCH_CAPTURE": str(self.patch_capture), + "KUBECTL_CALLED": str(self.kubectl_called), + "KSAIL_OUTPUT_PATH_LOG": str(self.output_path_log), + } + ) + environment.update(environment_overrides) + return subprocess.run( + [str(HELPER)], + cwd=ROOT, + env=environment, + text=True, + capture_output=True, + check=False, + ) + + @staticmethod + def _valid_config() -> dict[str, object]: + return { + "auths": { + "ghcr.io": { + "username": "devantler", + "password": "fixture-secret-token", + } + } + } + + def test_refreshes_only_the_root_secret_and_cleans_plaintext(self) -> None: + config = self._valid_config() + + result = self._run_helper(config) + + self.assertEqual(result.returncode, 0, result.stderr) + patch = json.loads(self.patch_capture.read_text(encoding="utf-8")) + encoded = patch["data"][".dockerconfigjson"] + + self.assertEqual(json.loads(base64.b64decode(encoded)), config) + temporary_config = Path(self.output_path_log.read_text(encoding="utf-8")) + self.assertFalse(temporary_config.exists()) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + + def test_missing_or_malformed_registry_auth_fails_closed(self) -> None: + invalid_configs: list[object] = [ + {"auths": {"ghcr.io": {"username": "devantler"}}}, + {"auths": {"ghcr.io": {"username": "", "password": "token"}}}, + {"auths": {}}, + "not-a-docker-config", + ] + + for config in invalid_configs: + with self.subTest(config=config): + result = self._run_helper(config) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.kubectl_called.exists()) + + def test_registry_denial_prevents_cluster_patch(self) -> None: + result = self._run_helper(self._valid_config(), FAKE_DOCKER_FAIL="true") + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.kubectl_called.exists()) + + def test_cluster_patch_failure_is_not_hidden(self) -> None: + result = self._run_helper(self._valid_config(), FAKE_KUBECTL_FAIL="true") + + self.assertEqual(result.returncode, 43) + self.assertTrue(self.kubectl_called.exists()) + + +class DeployActionOrderingTests(unittest.TestCase): + """Keep credential refreshes on both sides of KSail-owned mutations.""" + + def test_refresh_precedes_reconcile_and_is_reasserted_after_update(self) -> None: + action = ACTION.read_text(encoding="utf-8") + + first_refresh = action.index("id: refresh_flux_ghcr_auth") + reconcile = action.index("id: reconcile") + cluster_update = action.index( + "run: ksail --config ksail.prod.yaml cluster update" + ) + final_refresh = action.index("id: reassert_flux_ghcr_auth") + + self.assertLess(first_refresh, reconcile) + self.assertLess(reconcile, cluster_update) + self.assertLess(cluster_update, final_refresh) + self.assertIn( + "if: always() && steps.refresh_flux_ghcr_auth.outcome == 'success'", + action, + ) + self.assertEqual(action.count("scripts/refresh-flux-ghcr-auth.sh"), 2) + + def test_disaster_rebuild_refreshes_root_auth_before_reconcile(self) -> None: + workflow = DR_REBUILD.read_text(encoding="utf-8") + + refresh = workflow.index("scripts/refresh-flux-ghcr-auth.sh") + reconcile = workflow.index( + "run: ksail --config ksail.prod.yaml workload reconcile" + ) + + self.assertLess(refresh, reconcile) + + +if __name__ == "__main__": + unittest.main() From 43c6694ef43c7a6cf882bcf110b30b5d1819e016 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 13 Jul 2026 23:47:27 +0200 Subject: [PATCH 02/16] fix(deploy): harden GHCR credential preflight --- .github/actions/deploy-prod/action.yml | 20 ++-- AGENTS.md | 9 +- docs/dr/runbook.md | 9 +- scripts/refresh-flux-ghcr-auth.sh | 88 ++++++++++++++-- scripts/tests/test_refresh_flux_ghcr_auth.py | 100 +++++++++++++++++-- 5 files changed, 198 insertions(+), 28 deletions(-) diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index d6f64739c..3bd7465dd 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -126,6 +126,14 @@ runs: echo "::error::Prod cluster (context admin@prod) is unreachable with the restored kubeconfig (last kubectl error: ${last_err}). KUBE_CONFIG (and likely TALOS_CONFIG) are probably stale after a cluster rebuild — refresh both and update the 'prod' environment secrets. See docs/dr/runbook.md → 'Refresh CI deploy credentials'." exit 1 + - name: 🔎 Preflight Flux GHCR pull credential + id: preflight_flux_ghcr_auth + # Validate every shared pull scope BEFORE publishing latest. Do not patch + # yet: the matching artifact has not been published. This prevents a bad + # SOPS rotation from being discovered only after the mutable tag changes. + shell: bash + run: ./scripts/refresh-flux-ghcr-auth.sh --check-only + # Publish the GitOps manifests (push → sign → attest → reconcile) BEFORE # syncing Talos machine config (the "🔄 Update cluster" step runs LAST). # Manifest delivery never touches the nodes, so it must not be gated on @@ -284,11 +292,11 @@ runs: push-to-registry: true create-storage-record: false - - name: 🔑 Refresh Flux GHCR pull credential - id: refresh_flux_ghcr_auth - # Git/SOPS is authoritative for pulls. Refresh the KSail-managed root - # Secret before reconciliation so a rotation cannot deadlock Flux on the - # stale credential it needs to fetch the artifact containing the new one. + - name: 🔑 Reassert Flux GHCR pull credential after publish + id: reassert_flux_ghcr_auth_after_push + # Revalidate the newly-published artifact and reassert Git/SOPS before + # reconciliation. The initial preflight closed the mutable-tag failure + # window; this check proves the new tag is readable too. shell: bash run: ./scripts/refresh-flux-ghcr-auth.sh @@ -336,6 +344,6 @@ runs: # unrelated configuration change triggers setup. Reassert Git/SOPS even # when that update fails part-way, provided the source was already proven # valid by the first refresh. - if: always() && steps.refresh_flux_ghcr_auth.outcome == 'success' + if: always() && steps.preflight_flux_ghcr_auth.outcome == 'success' shell: bash run: ./scripts/refresh-flux-ghcr-auth.sh diff --git a/AGENTS.md b/AGENTS.md index 70f14f0d1..61cdcdf10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,9 +144,10 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 2. The `deploy-prod` composite action (shared by both paths) uses `ksail --config ksail.prod.yaml` to target the committed prod config. 3. `ksail.prod.yaml` has `kustomizationFile: clusters/prod`, so KSail/Flux use `k8s/clusters/prod/kustomization.yaml` as the entry point — no root `k8s/kustomization.yaml` or file rewriting is needed. 4. `ksail --config ksail.prod.yaml cluster create` (first run) or `cluster update` (subsequent runs) provisions / reconciles the Hetzner servers, Talos, CCM, and CSI. -5. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. -6. `scripts/refresh-flux-ghcr-auth.sh` decrypts only the Git/SOPS pull credential, proves it can read the artifact, and refreshes the KSail-managed root Flux Secret before reconciliation. -7. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. +5. `scripts/refresh-flux-ghcr-auth.sh --check-only` decrypts only the Git/SOPS pull credential and proves it can read Platform plus both private tenant packages before the mutable `latest` tag is published; it does not mutate the cluster. +6. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. +7. The bridge revalidates the newly-published artifact and reasserts root auth before reconciliation. +8. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. **Key differences from local:** @@ -164,7 +165,7 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o - **`ci.yaml`** — runs on `pull_request` (static manifest validation + Kubescape scan, no cluster) and `merge_group` (deploys prod via the Hetzner provider). Concurrency is shared with `cd.yaml` so a manual deploy and a merge-queue deploy can never run against the prod cluster at the same time. - **`cd.yaml`** — runs on `workflow_dispatch` (manual). Deploys to the production Hetzner cluster using `ksail --config ksail.prod.yaml`. Covers direct pushes to `main`, which bypass the merge queue and so are not deployed by `ci.yaml`. -- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (push → cosign-sign → attest SBOM + SLSA provenance → refresh root GHCR auth → Flux reconcile → Talos `cluster update` → reassert root GHCR auth), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. +- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (preflight root GHCR auth → push → cosign-sign → attest SBOM + SLSA provenance → revalidate/patch root auth → Flux reconcile → Talos `cluster update` → final reassert), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. **Required GitHub Secrets:** diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index ccca2f9f8..f44373055 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -500,9 +500,12 @@ single-source consolidation stays tracked by #2613. The production deploy closes the bootstrap loop in this order: -1. Push and sign the new artifact with `GHCR_TOKEN`. -2. Decrypt only `ghcr_dockerconfigjson`, verify it can pull the production - artifact, and patch `flux-system/ksail-registry-credentials`. +1. Decrypt only `ghcr_dockerconfigjson` and verify it can pull Platform plus both + private tenant packages before changing the mutable `latest` tag; this + preflight does not mutate the cluster. +2. Push and sign the new artifact with `GHCR_TOKEN`, then revalidate the SOPS + credential against the newly-published artifact and patch + `flux-system/ksail-registry-credentials`. 3. Reconcile Flux. The bootstrap Kustomization applies the same value to `variables-base`; `seed-ghcr` pushes it to OpenBao and tenant ExternalSecrets fan it out as `ghcr-auth`. diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index 9c127d101..b500cf379 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -7,9 +7,27 @@ set -euo pipefail +check_only=false +if (($# > 1)); then + echo "Usage: $0 [--check-only]" >&2 + exit 64 +fi +if (($# == 1)); then + if [[ "$1" != "--check-only" ]]; then + echo "Usage: $0 [--check-only]" >&2 + exit 64 + fi + check_only=true +fi + readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" readonly IMAGE="${FLUX_GHCR_IMAGE:-ghcr.io/devantler-tech/platform/manifests:latest}" readonly KUBE_CONTEXT="${KUBE_CONTEXT:-admin@prod}" +readonly -a REQUIRED_PULL_REPOSITORIES=( + "devantler-tech/platform/manifests" + "devantler-tech/wedding-app/manifests" + "devantler-tech/ascoachingogvaner/manifests" +) work_dir="$(mktemp -d)" trap 'rm -rf "${work_dir}"' EXIT @@ -17,6 +35,7 @@ chmod 700 "${work_dir}" umask 077 docker_config="${work_dir}/config.json" +curl_config="${work_dir}/curl.config" patch_file="${work_dir}/patch.json" # KSail embeds SOPS, so the deploy uses the same pinned toolchain as workload @@ -30,22 +49,77 @@ ksail workload cipher decrypt \ chmod 600 "${docker_config}" if ! jq -e ' - (.auths["ghcr.io"] // {}) - | (.username | type == "string" and length > 0) - and (.password | type == "string" and length > 0) + def non_empty_string: type == "string" and length > 0; + (.auths["ghcr.io"] // {}) as $auth + | ( + (($auth.username | non_empty_string) + and ($auth.password | non_empty_string)) + or + (try ( + $auth.auth + | @base64d + | capture("^(?[^:]+):(?.+)$") + | ((.username | non_empty_string) + and (.password | non_empty_string)) + ) catch false) + ) ' "${docker_config}" >/dev/null; then - echo "::error::The SOPS GHCR pull credential is not a valid Docker config with non-empty ghcr.io username/password fields." + echo "::error::The SOPS GHCR pull credential is not a valid Docker config with non-empty ghcr.io username/password or auth fields." exit 1 fi -# Prove the decrypted credential can actually pull the production artifact -# before mutating the cluster. DOCKER_CONFIG isolates it from the workflow's -# separate push credential in ~/.docker/config.json. +# Build curl's Basic-auth config without putting the credential in argv or +# stdout. Support both Docker config representations used in this repository: +# explicit username/password and base64(username:password) in auth. +jq -r ' + def non_empty_string: type == "string" and length > 0; + (.auths["ghcr.io"] // {}) as $auth + | if (($auth.username | non_empty_string) + and ($auth.password | non_empty_string)) then + "user = " + (($auth.username + ":" + $auth.password) | @json) + else + "user = " + (($auth.auth | @base64d) | @json) + end +' "${docker_config}" > "${curl_config}" +chmod 600 "${curl_config}" + +# The same pull credential fans out to Platform and both private tenant OCI +# sources. Prove GHCR grants every required package scope before changing the +# cluster; checking only Platform would let a partially-authorized rotation +# break the tenants later. +for repository in "${REQUIRED_PULL_REPOSITORIES[@]}"; do + if ! http_status="$(curl \ + --config "${curl_config}" \ + --silent \ + --show-error \ + --output /dev/null \ + --write-out '%{http_code}' \ + --get \ + --data-urlencode 'service=ghcr.io' \ + --data-urlencode "scope=repository:${repository}:pull" \ + 'https://ghcr.io/token')"; then + echo "::error::Could not validate the SOPS GHCR pull credential for ${repository}; the root Flux Secret was not changed." + exit 1 + fi + if [[ "${http_status}" != "200" ]]; then + echo "::error::The SOPS GHCR pull credential cannot read ${repository} (GHCR HTTP ${http_status}); the root Flux Secret was not changed." + exit 1 + fi +done + +# Also prove the production artifact exists and can be resolved through a real +# OCI client. DOCKER_CONFIG isolates this check from the workflow's separate +# push credential in ~/.docker/config.json. if ! DOCKER_CONFIG="${work_dir}" docker buildx imagetools inspect "${IMAGE}" >/dev/null; then echo "::error::The SOPS GHCR pull credential cannot read ${IMAGE}; the root Flux Secret was not changed." exit 1 fi +if [[ "${check_only}" == "true" ]]; then + echo "✅ Validated every required GHCR pull scope from Git/SOPS." + exit 0 +fi + # Merge only the Secret data field so KSail's ownership label and any unrelated # metadata survive. The sensitive payload stays in the pipe/temp file and never # appears in argv or logs. diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index 3aff6e5e7..fdf25b262 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -31,6 +31,7 @@ def setUp(self) -> None: self.patch_capture = self.workspace / "patch.json" self.kubectl_called = self.workspace / "kubectl-called" self.output_path_log = self.workspace / "ksail-output-path" + self.curl_scope_log = self.workspace / "curl-scopes" self._write_executable( "ksail", @@ -73,6 +74,36 @@ def setUp(self) -> None: fi """, ) + self._write_executable( + "curl", + """ + #!/usr/bin/env bash + set -euo pipefail + config="" + scope="" + while (($#)); do + case "$1" in + --config) + config="$2" + shift 2 + ;; + --data-urlencode) + case "$2" in scope=*) scope="${2#scope=}" ;; esac + shift 2 + ;; + *) shift ;; + esac + done + test -f "$config" + test -n "$scope" + printf '%s\n' "$scope" >> "$CURL_SCOPE_LOG" + if [[ "$scope" == *"${FAKE_CURL_DENY_REPOSITORY:-disabled}"* ]]; then + printf '403' + else + printf '200' + fi + """, + ) self._write_executable( "kubectl", """ @@ -106,10 +137,18 @@ def _write_executable(self, name: str, body: str) -> None: path.chmod(0o755) def _run_helper( - self, config: object, **environment_overrides: str + self, + config: object, + helper_args: tuple[str, ...] = (), + **environment_overrides: str, ) -> subprocess.CompletedProcess[str]: self.decrypted_config.write_text(json.dumps(config), encoding="utf-8") - for marker in (self.patch_capture, self.kubectl_called, self.output_path_log): + for marker in ( + self.patch_capture, + self.kubectl_called, + self.output_path_log, + self.curl_scope_log, + ): marker.unlink(missing_ok=True) environment = os.environ.copy() environment.update( @@ -119,11 +158,12 @@ def _run_helper( "PATCH_CAPTURE": str(self.patch_capture), "KUBECTL_CALLED": str(self.kubectl_called), "KSAIL_OUTPUT_PATH_LOG": str(self.output_path_log), + "CURL_SCOPE_LOG": str(self.curl_scope_log), } ) environment.update(environment_overrides) return subprocess.run( - [str(HELPER)], + [str(HELPER), *helper_args], cwd=ROOT, env=environment, text=True, @@ -155,11 +195,38 @@ def test_refreshes_only_the_root_secret_and_cleans_plaintext(self) -> None: temporary_config = Path(self.output_path_log.read_text(encoding="utf-8")) self.assertFalse(temporary_config.exists()) self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + self.assertEqual( + self.curl_scope_log.read_text(encoding="utf-8").splitlines(), + [ + "repository:devantler-tech/platform/manifests:pull", + "repository:devantler-tech/wedding-app/manifests:pull", + "repository:devantler-tech/ascoachingogvaner/manifests:pull", + ], + ) + + def test_accepts_standard_auth_only_docker_config(self) -> None: + auth = base64.b64encode(b"devantler:fixture-secret-token").decode() + config = {"auths": {"ghcr.io": {"auth": auth}}} + + result = self._run_helper(config) + + self.assertEqual(result.returncode, 0, result.stderr) + patch = json.loads(self.patch_capture.read_text(encoding="utf-8")) + encoded = patch["data"][".dockerconfigjson"] + self.assertEqual(json.loads(base64.b64decode(encoded)), config) + + def test_check_only_preflights_without_patching(self) -> None: + result = self._run_helper(self._valid_config(), ("--check-only",)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(self.kubectl_called.exists()) + self.assertFalse(self.patch_capture.exists()) def test_missing_or_malformed_registry_auth_fails_closed(self) -> None: invalid_configs: list[object] = [ {"auths": {"ghcr.io": {"username": "devantler"}}}, {"auths": {"ghcr.io": {"username": "", "password": "token"}}}, + {"auths": {"ghcr.io": {"auth": "not-base64"}}}, {"auths": {}}, "not-a-docker-config", ] @@ -176,6 +243,15 @@ def test_registry_denial_prevents_cluster_patch(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertFalse(self.kubectl_called.exists()) + def test_tenant_package_denial_prevents_cluster_patch(self) -> None: + result = self._run_helper( + self._valid_config(), + FAKE_CURL_DENY_REPOSITORY="devantler-tech/wedding-app/manifests", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.kubectl_called.exists()) + def test_cluster_patch_failure_is_not_hidden(self) -> None: result = self._run_helper(self._valid_config(), FAKE_KUBECTL_FAIL="true") @@ -189,21 +265,29 @@ class DeployActionOrderingTests(unittest.TestCase): def test_refresh_precedes_reconcile_and_is_reasserted_after_update(self) -> None: action = ACTION.read_text(encoding="utf-8") - first_refresh = action.index("id: refresh_flux_ghcr_auth") + first_refresh = action.index("id: preflight_flux_ghcr_auth") + push = action.index("run: ksail --config ksail.prod.yaml workload push") + post_push_refresh = action.index("id: reassert_flux_ghcr_auth_after_push") reconcile = action.index("id: reconcile") cluster_update = action.index( "run: ksail --config ksail.prod.yaml cluster update" ) - final_refresh = action.index("id: reassert_flux_ghcr_auth") + final_refresh = action.index("id: reassert_flux_ghcr_auth\n") - self.assertLess(first_refresh, reconcile) + self.assertLess(first_refresh, push) + self.assertLess(push, post_push_refresh) + self.assertLess(post_push_refresh, reconcile) self.assertLess(reconcile, cluster_update) self.assertLess(cluster_update, final_refresh) self.assertIn( - "if: always() && steps.refresh_flux_ghcr_auth.outcome == 'success'", + "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only", + action, + ) + self.assertIn( + "if: always() && steps.preflight_flux_ghcr_auth.outcome == 'success'", action, ) - self.assertEqual(action.count("scripts/refresh-flux-ghcr-auth.sh"), 2) + self.assertEqual(action.count("scripts/refresh-flux-ghcr-auth.sh"), 3) def test_disaster_rebuild_refreshes_root_auth_before_reconcile(self) -> None: workflow = DR_REBUILD.read_text(encoding="utf-8") From 06c5e81e4f0a4f04ab27625e1a92548955889e9f Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 00:05:07 +0200 Subject: [PATCH 03/16] fix(deploy): verify every GHCR pull consumer --- .github/actions/deploy-prod/action.yml | 7 +- .github/workflows/dr-rebuild.yaml | 5 + AGENTS.md | 2 +- docs/dr/runbook.md | 18 ++-- scripts/refresh-flux-ghcr-auth.sh | 67 ++++++++---- scripts/tests/test_refresh_flux_ghcr_auth.py | 105 +++++++++++++------ 6 files changed, 140 insertions(+), 64 deletions(-) diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 3bd7465dd..62aa84251 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -128,9 +128,10 @@ runs: - name: 🔎 Preflight Flux GHCR pull credential id: preflight_flux_ghcr_auth - # Validate every shared pull scope BEFORE publishing latest. Do not patch - # yet: the matching artifact has not been published. This prevents a bad - # SOPS rotation from being discovered only after the mutable tag changes. + # Perform a real OCI read for all five packages served by this credential + # BEFORE publishing latest. Do not patch yet: the matching artifact has + # not been published. This prevents a bad SOPS rotation from being + # discovered only after the mutable tag changes. shell: bash run: ./scripts/refresh-flux-ghcr-auth.sh --check-only diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index 60337fff5..a9bf94d68 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -94,6 +94,11 @@ jobs: echo "${SOPS_AGE_KEY}" > ~/.config/sops/age/keys.txt chmod 600 ~/.config/sops/age/keys.txt + - name: 🔎 Preflight Flux GHCR pull credential + # Validate every package served by the shared Git/SOPS credential + # before creating infrastructure or publishing a mutable latest tag. + run: ./scripts/refresh-flux-ghcr-auth.sh --check-only + - name: 🏗️ Create cluster # From-zero provisioning: Hetzner servers, Talos boot, CCM, CSI. Writes # a fresh kubeconfig (~/.kube/config, context admin@prod) and diff --git a/AGENTS.md b/AGENTS.md index 61cdcdf10..f10855188 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,7 +144,7 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 2. The `deploy-prod` composite action (shared by both paths) uses `ksail --config ksail.prod.yaml` to target the committed prod config. 3. `ksail.prod.yaml` has `kustomizationFile: clusters/prod`, so KSail/Flux use `k8s/clusters/prod/kustomization.yaml` as the entry point — no root `k8s/kustomization.yaml` or file rewriting is needed. 4. `ksail --config ksail.prod.yaml cluster create` (first run) or `cluster update` (subsequent runs) provisions / reconciles the Hetzner servers, Talos, CCM, and CSI. -5. `scripts/refresh-flux-ghcr-auth.sh --check-only` decrypts only the Git/SOPS pull credential and proves it can read Platform plus both private tenant packages before the mutable `latest` tag is published; it does not mutate the cluster. +5. `scripts/refresh-flux-ghcr-auth.sh --check-only` decrypts only the Git/SOPS pull credential and performs real OCI manifest reads for all five private consumers (the Platform and tenant manifest artifacts plus both tenant application images) before a mutable `latest` tag is published; the DR workflow also runs it before creating infrastructure. It does not mutate the cluster. 6. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. 7. The bridge revalidates the newly-published artifact and reasserts root auth before reconciliation. 8. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index f44373055..fa9950e49 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -159,15 +159,19 @@ export HCLOUD_TOKEN= export GHCR_TOKEN= export SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt # points at the env's Age key -# 2. Boot a fresh cluster (ksail handles Talos boot, CCM, CSI, kubeconfig) +# 2. Prove the Git/SOPS pull credential can read every private package before +# creating infrastructure or publishing a mutable latest tag +./scripts/refresh-flux-ghcr-auth.sh --check-only + +# 3. Boot a fresh cluster (ksail handles Talos boot, CCM, CSI, kubeconfig) ksail --config ksail.prod.yaml cluster create -# 3. Bootstrap Flux from this repo +# 4. Bootstrap Flux from this repo ksail --config ksail.prod.yaml workload push # packages -> GHCR ./scripts/refresh-flux-ghcr-auth.sh # Git/SOPS -> root Flux auth ksail --config ksail.prod.yaml workload reconcile # Flux pulls and applies -# 4. Wait for Flux to settle +# 5. Wait for Flux to settle flux get kustomizations -A # Re-run if any are NotReady; expect convergence in 10-15 minutes @@ -500,9 +504,11 @@ single-source consolidation stays tracked by #2613. The production deploy closes the bootstrap loop in this order: -1. Decrypt only `ghcr_dockerconfigjson` and verify it can pull Platform plus both - private tenant packages before changing the mutable `latest` tag; this - preflight does not mutate the cluster. +1. Decrypt only `ghcr_dockerconfigjson` and perform real OCI manifest reads for + all five private consumers: Platform manifests, both tenant manifest + artifacts, and both tenant application images. This happens before changing + a mutable `latest` tag (and before infrastructure creation during DR), and + does not mutate the cluster. 2. Push and sign the new artifact with `GHCR_TOKEN`, then revalidate the SOPS credential against the newly-published artifact and patch `flux-system/ksail-registry-credentials`. diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index b500cf379..6c3a07cbb 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -21,12 +21,13 @@ if (($# == 1)); then fi readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" -readonly IMAGE="${FLUX_GHCR_IMAGE:-ghcr.io/devantler-tech/platform/manifests:latest}" readonly KUBE_CONTEXT="${KUBE_CONTEXT:-admin@prod}" readonly -a REQUIRED_PULL_REPOSITORIES=( "devantler-tech/platform/manifests" "devantler-tech/wedding-app/manifests" "devantler-tech/ascoachingogvaner/manifests" + "devantler-tech/wedding-app" + "devantler-tech/ascoachingogvaner" ) work_dir="$(mktemp -d)" @@ -35,7 +36,9 @@ chmod 700 "${work_dir}" umask 077 docker_config="${work_dir}/config.json" -curl_config="${work_dir}/curl.config" +basic_curl_config="${work_dir}/curl-basic.config" +bearer_curl_config="${work_dir}/curl-bearer.config" +token_response="${work_dir}/token.json" patch_file="${work_dir}/patch.json" # KSail embeds SOPS, so the deploy uses the same pinned toolchain as workload @@ -80,43 +83,63 @@ jq -r ' else "user = " + (($auth.auth | @base64d) | @json) end -' "${docker_config}" > "${curl_config}" -chmod 600 "${curl_config}" +' "${docker_config}" > "${basic_curl_config}" +chmod 600 "${basic_curl_config}" -# The same pull credential fans out to Platform and both private tenant OCI -# sources. Prove GHCR grants every required package scope before changing the -# cluster; checking only Platform would let a partially-authorized rotation -# break the tenants later. +# The same pull credential fans out to Flux OCI sources and private tenant +# workloads. GHCR permissions are package-granular, and the token endpoint can +# return only the intersection of requested and granted scopes. Therefore a +# token HTTP 200 is not proof of pull access: exchange it for a bearer token, +# then perform a real registry manifest GET for every package. Both credentials +# stay in mode-0600 files. --disable must remain curl's first argument so an +# ambient ~/.curlrc cannot enable tracing, add URLs, or otherwise expose auth. for repository in "${REQUIRED_PULL_REPOSITORIES[@]}"; do - if ! http_status="$(curl \ - --config "${curl_config}" \ + if ! http_status="$(curl --disable \ + --config "${basic_curl_config}" \ --silent \ --show-error \ - --output /dev/null \ + --output "${token_response}" \ --write-out '%{http_code}' \ --get \ --data-urlencode 'service=ghcr.io' \ --data-urlencode "scope=repository:${repository}:pull" \ 'https://ghcr.io/token')"; then - echo "::error::Could not validate the SOPS GHCR pull credential for ${repository}; the root Flux Secret was not changed." + echo "::error::Could not request a GHCR pull token for ${repository}; the root Flux Secret was not changed." + exit 1 + fi + if [[ "${http_status}" != "200" ]] || ! jq -e ' + (.token // .access_token // "") + | type == "string" and length > 0 + ' "${token_response}" >/dev/null; then + echo "::error::The SOPS GHCR credential could not obtain a pull token for ${repository} (GHCR HTTP ${http_status}); the root Flux Secret was not changed." + exit 1 + fi + + jq -r ' + (.token // .access_token) as $token + | "header = " + (("Authorization: Bearer " + $token) | @json) + ' "${token_response}" > "${bearer_curl_config}" + chmod 600 "${bearer_curl_config}" + + if ! http_status="$(curl --disable \ + --config "${bearer_curl_config}" \ + --silent \ + --show-error \ + --output /dev/null \ + --write-out '%{http_code}' \ + --header 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json' \ + "https://ghcr.io/v2/${repository}/manifests/latest")"; then + echo "::error::Could not read the GHCR manifest for ${repository}; the root Flux Secret was not changed." exit 1 fi if [[ "${http_status}" != "200" ]]; then - echo "::error::The SOPS GHCR pull credential cannot read ${repository} (GHCR HTTP ${http_status}); the root Flux Secret was not changed." + echo "::error::The SOPS GHCR pull credential cannot read ${repository}:latest (GHCR HTTP ${http_status}); the root Flux Secret was not changed." exit 1 fi done -# Also prove the production artifact exists and can be resolved through a real -# OCI client. DOCKER_CONFIG isolates this check from the workflow's separate -# push credential in ~/.docker/config.json. -if ! DOCKER_CONFIG="${work_dir}" docker buildx imagetools inspect "${IMAGE}" >/dev/null; then - echo "::error::The SOPS GHCR pull credential cannot read ${IMAGE}; the root Flux Secret was not changed." - exit 1 -fi - if [[ "${check_only}" == "true" ]]; then - echo "✅ Validated every required GHCR pull scope from Git/SOPS." + echo "✅ Validated every required GHCR package pull from Git/SOPS." exit 0 fi diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index fdf25b262..fc136eecf 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -31,7 +31,7 @@ def setUp(self) -> None: self.patch_capture = self.workspace / "patch.json" self.kubectl_called = self.workspace / "kubectl-called" self.output_path_log = self.workspace / "ksail-output-path" - self.curl_scope_log = self.workspace / "curl-scopes" + self.registry_read_log = self.workspace / "registry-reads" self._write_executable( "ksail", @@ -58,46 +58,65 @@ def setUp(self) -> None: cp "$FAKE_DECRYPTED_CONFIG" "$output" """, ) - self._write_executable( - "docker", - """ - #!/usr/bin/env bash - set -euo pipefail - test "$1" = buildx - test "$2" = imagetools - test "$3" = inspect - test "$4" = ghcr.io/devantler-tech/platform/manifests:latest - test -f "$DOCKER_CONFIG/config.json" - if [[ "${FAKE_DOCKER_FAIL:-false}" == true ]]; then - echo 'registry pull preflight denied' >&2 - exit 42 - fi - """, - ) self._write_executable( "curl", """ #!/usr/bin/env bash set -euo pipefail + test "$1" = --disable + shift config="" + output="" scope="" + url="" while (($#)); do case "$1" in --config) config="$2" shift 2 ;; + --output) + output="$2" + shift 2 + ;; --data-urlencode) case "$2" in scope=*) scope="${2#scope=}" ;; esac shift 2 ;; - *) shift ;; + --write-out|--header) + shift 2 + ;; + --silent|--show-error|--get) + shift + ;; + https://*) + url="$1" + shift + ;; + *) + echo "unexpected curl argument: $1" >&2 + exit 90 + ;; esac done test -f "$config" - test -n "$scope" - printf '%s\n' "$scope" >> "$CURL_SCOPE_LOG" - if [[ "$scope" == *"${FAKE_CURL_DENY_REPOSITORY:-disabled}"* ]]; then + test -n "$output" + test -n "$url" + + if [[ "$url" == https://ghcr.io/token ]]; then + test -n "$scope" + grep -q '^user = ' "$config" + printf '{"token":"fixture-registry-token"}' > "$output" + printf '200' + exit 0 + fi + + repository="${url#https://ghcr.io/v2/}" + repository="${repository%/manifests/latest}" + test "$repository" != "$url" + grep -q 'Authorization: Bearer fixture-registry-token' "$config" + printf '%s\n' "$repository" >> "$REGISTRY_READ_LOG" + if [[ "$repository" == "${FAKE_CURL_DENY_REPOSITORY:-disabled}" ]]; then printf '403' else printf '200' @@ -147,7 +166,7 @@ def _run_helper( self.patch_capture, self.kubectl_called, self.output_path_log, - self.curl_scope_log, + self.registry_read_log, ): marker.unlink(missing_ok=True) environment = os.environ.copy() @@ -158,7 +177,7 @@ def _run_helper( "PATCH_CAPTURE": str(self.patch_capture), "KUBECTL_CALLED": str(self.kubectl_called), "KSAIL_OUTPUT_PATH_LOG": str(self.output_path_log), - "CURL_SCOPE_LOG": str(self.curl_scope_log), + "REGISTRY_READ_LOG": str(self.registry_read_log), } ) environment.update(environment_overrides) @@ -196,11 +215,13 @@ def test_refreshes_only_the_root_secret_and_cleans_plaintext(self) -> None: self.assertFalse(temporary_config.exists()) self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) self.assertEqual( - self.curl_scope_log.read_text(encoding="utf-8").splitlines(), + self.registry_read_log.read_text(encoding="utf-8").splitlines(), [ - "repository:devantler-tech/platform/manifests:pull", - "repository:devantler-tech/wedding-app/manifests:pull", - "repository:devantler-tech/ascoachingogvaner/manifests:pull", + "devantler-tech/platform/manifests", + "devantler-tech/wedding-app/manifests", + "devantler-tech/ascoachingogvaner/manifests", + "devantler-tech/wedding-app", + "devantler-tech/ascoachingogvaner", ], ) @@ -238,15 +259,20 @@ def test_missing_or_malformed_registry_auth_fails_closed(self) -> None: self.assertFalse(self.kubectl_called.exists()) def test_registry_denial_prevents_cluster_patch(self) -> None: - result = self._run_helper(self._valid_config(), FAKE_DOCKER_FAIL="true") + result = self._run_helper( + self._valid_config(), + FAKE_CURL_DENY_REPOSITORY="devantler-tech/platform/manifests", + ) self.assertNotEqual(result.returncode, 0) self.assertFalse(self.kubectl_called.exists()) - def test_tenant_package_denial_prevents_cluster_patch(self) -> None: + def test_token_success_without_registry_read_access_prevents_cluster_patch( + self, + ) -> None: result = self._run_helper( self._valid_config(), - FAKE_CURL_DENY_REPOSITORY="devantler-tech/wedding-app/manifests", + FAKE_CURL_DENY_REPOSITORY="devantler-tech/wedding-app", ) self.assertNotEqual(result.returncode, 0) @@ -289,15 +315,30 @@ def test_refresh_precedes_reconcile_and_is_reasserted_after_update(self) -> None ) self.assertEqual(action.count("scripts/refresh-flux-ghcr-auth.sh"), 3) - def test_disaster_rebuild_refreshes_root_auth_before_reconcile(self) -> None: + def test_disaster_rebuild_preflights_before_mutation_and_refreshes_before_reconcile( + self, + ) -> None: workflow = DR_REBUILD.read_text(encoding="utf-8") - refresh = workflow.index("scripts/refresh-flux-ghcr-auth.sh") + preflight = workflow.index( + "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only" + ) + cluster_create = workflow.index( + "run: ksail --config ksail.prod.yaml cluster create" + ) + push = workflow.index("run: ksail --config ksail.prod.yaml workload push") + refresh = workflow.index( + "run: ./scripts/refresh-flux-ghcr-auth.sh\n" + ) reconcile = workflow.index( "run: ksail --config ksail.prod.yaml workload reconcile" ) + self.assertLess(preflight, cluster_create) + self.assertLess(cluster_create, push) + self.assertLess(push, refresh) self.assertLess(refresh, reconcile) + self.assertEqual(workflow.count("scripts/refresh-flux-ghcr-auth.sh"), 2) if __name__ == "__main__": From 4e96c32e842ead97fe889ec54202f141a326f5ab Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 00:16:14 +0200 Subject: [PATCH 04/16] fix(deploy): gate final auth reassert --- .github/actions/deploy-prod/action.yml | 13 +++++++++---- scripts/tests/test_refresh_flux_ghcr_auth.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 62aa84251..f055cdd9d 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -342,9 +342,14 @@ runs: - name: 🔑 Reassert Flux GHCR pull credential id: reassert_flux_ghcr_auth # cluster update can rewrite KSail's root Secret from GHCR_TOKEN when an - # unrelated configuration change triggers setup. Reassert Git/SOPS even - # when that update fails part-way, provided the source was already proven - # valid by the first refresh. - if: always() && steps.preflight_flux_ghcr_auth.outcome == 'success' + # unrelated configuration change triggers setup. Reassert Git/SOPS when + # that update fails part-way, but only after the publish, signing, + # attestation, post-publish auth refresh, and Flux reconcile all + # succeeded. A failed delivery must never expose a partial latest tag to + # Flux's normal polling loop. + if: >- + always() && + steps.reassert_flux_ghcr_auth_after_push.outcome == 'success' && + steps.reconcile.outcome == 'success' shell: bash run: ./scripts/refresh-flux-ghcr-auth.sh diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index fc136eecf..cf082c1bb 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -309,9 +309,15 @@ def test_refresh_precedes_reconcile_and_is_reasserted_after_update(self) -> None "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only", action, ) + final_refresh_step = action[final_refresh:] + self.assertIn("always() &&", final_refresh_step) self.assertIn( - "if: always() && steps.preflight_flux_ghcr_auth.outcome == 'success'", - action, + "steps.reassert_flux_ghcr_auth_after_push.outcome == 'success'", + final_refresh_step, + ) + self.assertIn( + "steps.reconcile.outcome == 'success'", + final_refresh_step, ) self.assertEqual(action.count("scripts/refresh-flux-ghcr-auth.sh"), 3) From b70d68842dcdd55ab3a0958a3099475e68389ce8 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 00:55:03 +0200 Subject: [PATCH 05/16] fix(deploy): synchronize GHCR pull consumers --- .github/actions/deploy-prod/action.yml | 21 ++- AGENTS.md | 11 +- docs/dr/runbook.md | 33 ++-- .../vault-seed/push-secret-seed-ghcr.yaml | 5 +- scripts/refresh-flux-ghcr-auth.sh | 161 ++++++++++++++-- scripts/tests/test_refresh_flux_ghcr_auth.py | 173 +++++++++++++++++- 6 files changed, 349 insertions(+), 55 deletions(-) diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index f055cdd9d..47747f1fb 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -128,7 +128,7 @@ runs: - name: 🔎 Preflight Flux GHCR pull credential id: preflight_flux_ghcr_auth - # Perform a real OCI read for all five packages served by this credential + # Perform a real OCI read for all six packages served by this credential # BEFORE publishing latest. Do not patch yet: the matching artifact has # not been published. This prevents a bad SOPS rotation from being # discovered only after the mutable tag changes. @@ -295,9 +295,10 @@ runs: - name: 🔑 Reassert Flux GHCR pull credential after publish id: reassert_flux_ghcr_auth_after_push - # Revalidate the newly-published artifact and reassert Git/SOPS before - # reconciliation. The initial preflight closed the mutable-tag failure - # window; this check proves the new tag is readable too. + # Revalidate the newly-published artifact, reassert root auth, and force + # the variables-base -> OpenBao -> tenant/Kyverno fan-out before apps + # reconcile. The initial preflight closed the mutable-tag failure window; + # this check proves the new tag and every private consumer are readable. shell: bash run: ./scripts/refresh-flux-ghcr-auth.sh @@ -339,14 +340,14 @@ runs: WG_SERVER_PRIVATE_KEY: ${{ inputs.wg-server-private-key }} run: ksail --config ksail.prod.yaml cluster update - - name: 🔑 Reassert Flux GHCR pull credential + - name: 🔑 Reassert Flux and consumer GHCR pull credential id: reassert_flux_ghcr_auth # cluster update can rewrite KSail's root Secret from GHCR_TOKEN when an - # unrelated configuration change triggers setup. Reassert Git/SOPS when - # that update fails part-way, but only after the publish, signing, - # attestation, post-publish auth refresh, and Flux reconcile all - # succeeded. A failed delivery must never expose a partial latest tag to - # Flux's normal polling loop. + # unrelated configuration change triggers setup. Reassert Git/SOPS and + # its synchronous consumer fan-out when that update fails part-way, but + # only after the publish, signing, attestation, post-publish auth refresh, + # and Flux reconcile all succeeded. A failed delivery must never expose a + # partial latest tag to Flux's normal polling loop. if: >- always() && steps.reassert_flux_ghcr_auth_after_push.outcome == 'success' && diff --git a/AGENTS.md b/AGENTS.md index f10855188..601012857 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,9 +144,9 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 2. The `deploy-prod` composite action (shared by both paths) uses `ksail --config ksail.prod.yaml` to target the committed prod config. 3. `ksail.prod.yaml` has `kustomizationFile: clusters/prod`, so KSail/Flux use `k8s/clusters/prod/kustomization.yaml` as the entry point — no root `k8s/kustomization.yaml` or file rewriting is needed. 4. `ksail --config ksail.prod.yaml cluster create` (first run) or `cluster update` (subsequent runs) provisions / reconciles the Hetzner servers, Talos, CCM, and CSI. -5. `scripts/refresh-flux-ghcr-auth.sh --check-only` decrypts only the Git/SOPS pull credential and performs real OCI manifest reads for all five private consumers (the Platform and tenant manifest artifacts plus both tenant application images) before a mutable `latest` tag is published; the DR workflow also runs it before creating infrastructure. It does not mutate the cluster. +5. `scripts/refresh-flux-ghcr-auth.sh --check-only` decrypts only the Git/SOPS pull credential and performs real OCI manifest reads for all six private consumers (the Platform and tenant manifest artifacts, both tenant application images, and the KSail package used by Kyverno signature verification) before a mutable `latest` tag is published; the DR workflow also runs it before creating infrastructure. It does not mutate the cluster. 6. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. -7. The bridge revalidates the newly-published artifact and reasserts root auth before reconciliation. +7. The bridge revalidates the newly-published artifact, reasserts root auth, updates `variables-base`, and force-syncs the PushSecret plus tenant/Kyverno ExternalSecrets before reconciliation. 8. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. **Key differences from local:** @@ -178,9 +178,10 @@ The authoritative **Flux and tenant** GHCR pull credential is `k8s/bases/bootstrap/secret.enc.yaml`. The deploy bridge refreshes `flux-system/ksail-registry-credentials` from that value before Flux must fetch the artifact and reasserts it after `cluster update` in case KSail rewrites its -managed Secret. Flux then applies the same SOPS value to `variables-base`; the -`seed-ghcr` PushSecret fans it into OpenBao and tenant ExternalSecrets materialise -their `ghcr-auth` copies. A direct credential commit to `main` still needs a +managed Secret. On existing clusters the bridge also updates `variables-base`, +force-syncs `seed-ghcr` into OpenBao, force-syncs the tenant/Kyverno +ExternalSecrets, and verifies their materialised `ghcr-auth` payloads before +apps reconcile. A direct credential commit to `main` still needs a manual `CD` workflow dispatch because direct pushes bypass the merge-queue deploy. Talos node registry auth still derives from `GHCR_TOKEN`; consolidating that remaining pull path stays tracked by #2613 and the KSail credential work. diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index fa9950e49..73087e406 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -168,7 +168,7 @@ ksail --config ksail.prod.yaml cluster create # 4. Bootstrap Flux from this repo ksail --config ksail.prod.yaml workload push # packages -> GHCR -./scripts/refresh-flux-ghcr-auth.sh # Git/SOPS -> root Flux auth +./scripts/refresh-flux-ghcr-auth.sh # Git/SOPS -> root + existing fan-out ksail --config ksail.prod.yaml workload reconcile # Flux pulls and applies # 5. Wait for Flux to settle @@ -505,18 +505,23 @@ single-source consolidation stays tracked by #2613. The production deploy closes the bootstrap loop in this order: 1. Decrypt only `ghcr_dockerconfigjson` and perform real OCI manifest reads for - all five private consumers: Platform manifests, both tenant manifest - artifacts, and both tenant application images. This happens before changing - a mutable `latest` tag (and before infrastructure creation during DR), and - does not mutate the cluster. + all six private consumers: Platform manifests, both tenant manifest + artifacts, both tenant application images, and the KSail package used by + Kyverno signature verification. This happens before changing a mutable + `latest` tag (and before infrastructure creation during DR), and does not + mutate the cluster. 2. Push and sign the new artifact with `GHCR_TOKEN`, then revalidate the SOPS - credential against the newly-published artifact and patch - `flux-system/ksail-registry-credentials`. -3. Reconcile Flux. The bootstrap Kustomization applies the same value to - `variables-base`; `seed-ghcr` pushes it to OpenBao and tenant ExternalSecrets - fan it out as `ghcr-auth`. -4. Reassert the root Secret after `cluster update`, because KSail can rewrite its - managed Secret when another cluster setting changes. + credential against the newly-published artifact. Patch + `flux-system/ksail-registry-credentials` and, on an existing cluster, + `variables-base`; force-sync `seed-ghcr` into OpenBao; force-sync the + tenant/Kyverno ExternalSecrets; and verify every materialised `ghcr-auth` + payload matches Git/SOPS. +3. Reconcile Flux only after that synchronous fan-out succeeds. A fresh DR + cluster has no fan-out resources yet, so its first reconcile creates the + entire chain directly from the same current artifact. +4. Re-run the bridge after `cluster update`, because KSail can rewrite its + managed root Secret when another cluster setting changes; the bridge also + re-verifies the downstream fan-out. For a normal rotation, update the encrypted value through a PR and let the merge queue deploy it. If the encrypted file was pushed directly to `main`, manually @@ -532,8 +537,8 @@ flux reconcile source oci flux-system -n flux-system --context admin@prod flux reconcile kustomization flux-system -n flux-system --context admin@prod ``` -If tenant sources still show `DENIED`, force the existing fan-out after -`variables-base` has converged: +If tenant sources still show `DENIED` after an older/manual recovery path, force +the existing fan-out after `variables-base` has converged: ```bash stamp=$(date +%s) diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml index e02123e4c..c15587603 100644 --- a/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml +++ b/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml @@ -9,8 +9,9 @@ # (infra-ghcr-readonly) — a platform secret, not a tenant secret. Source key # `ghcr_dockerconfigjson` lives in secret.enc.yaml (shared, # cluster-independent — the same org token both clusters use) and holds the full -# `{"auths":{"ghcr.io":{...}}}` document. Re-pushed hourly, so a rotated -# token (or a re-initialized vault) converges without manual re-apply. +# `{"auths":{"ghcr.io":{...}}}` document. The production deploy bridge +# force-syncs this path before apps reconcile; the hourly interval is the +# fallback for out-of-band changes or a re-initialized vault. # # This one is deliberately SOPS-seeded, NOT user-fed, even though it is an # upstream token: the verify-image-signatures ClusterPolicy needs the kyverno diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index 6c3a07cbb..1de78110e 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -22,13 +22,27 @@ fi readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" readonly KUBE_CONTEXT="${KUBE_CONTEXT:-admin@prod}" +readonly SYNC_ATTEMPTS="${FLUX_GHCR_SYNC_ATTEMPTS:-60}" +readonly SYNC_INTERVAL="${FLUX_GHCR_SYNC_INTERVAL:-2}" readonly -a REQUIRED_PULL_REPOSITORIES=( "devantler-tech/platform/manifests" "devantler-tech/wedding-app/manifests" "devantler-tech/ascoachingogvaner/manifests" "devantler-tech/wedding-app" "devantler-tech/ascoachingogvaner" + "devantler-tech/ksail" ) +readonly -a FANOUT_NAMESPACES=( + "wedding-app" + "ascoachingogvaner" + "kyverno" +) + +if ! [[ "${SYNC_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]] \ + || ! [[ "${SYNC_INTERVAL}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + echo "::error::FLUX_GHCR_SYNC_ATTEMPTS and FLUX_GHCR_SYNC_INTERVAL must be non-negative numbers, with at least one attempt." + exit 64 +fi work_dir="$(mktemp -d)" trap 'rm -rf "${work_dir}"' EXIT @@ -40,6 +54,79 @@ basic_curl_config="${work_dir}/curl-basic.config" bearer_curl_config="${work_dir}/curl-bearer.config" token_response="${work_dir}/token.json" patch_file="${work_dir}/patch.json" +variables_patch_file="${work_dir}/variables-patch.json" +expected_normalized="${work_dir}/expected-normalized.json" + +force_sync_resource() { + local kind="$1" + local namespace="$2" + local name="$3" + local before_file="${work_dir}/${kind}-${namespace}-${name}-before.json" + local current_file="${work_dir}/${kind}-${namespace}-${name}-current.json" + local before_refresh + local attempt + local stamp + + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + get "${kind}" "${name}" \ + -o json \ + > "${before_file}" + before_refresh="$(jq -r '.status.refreshTime // ""' "${before_file}")" + stamp="$(date -u +%Y%m%dT%H%M%SZ)-$$-${RANDOM}" + + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + annotate "${kind}" "${name}" \ + "force-sync=${stamp}" \ + --overwrite \ + >/dev/null + + for ((attempt = 1; attempt <= SYNC_ATTEMPTS; attempt++)); do + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + get "${kind}" "${name}" \ + -o json \ + > "${current_file}" + if jq -e --arg before "${before_refresh}" ' + (.status.refreshTime // "") as $refresh + | ($refresh != "" and $refresh != $before) + and any(.status.conditions[]?; + .type == "Ready" and .status == "True") + ' "${current_file}" >/dev/null; then + return 0 + fi + sleep "${SYNC_INTERVAL}" + done + + echo "::error::Timed out waiting for ${kind}/${namespace}/${name} to complete the forced GHCR credential sync." + return 1 +} + +verify_consumer_secret() { + local namespace="$1" + local secret_file="${work_dir}/consumer-${namespace}.json" + local decoded_file="${work_dir}/consumer-${namespace}-decoded.json" + local normalized_file="${work_dir}/consumer-${namespace}-normalized.json" + + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + get secret ghcr-auth \ + -o json \ + > "${secret_file}" + if ! jq -er '.data[".dockerconfigjson"] | @base64d' \ + "${secret_file}" \ + > "${decoded_file}" 2>/dev/null \ + || ! jq -S -c . "${decoded_file}" > "${normalized_file}" 2>/dev/null \ + || ! cmp -s "${expected_normalized}" "${normalized_file}"; then + echo "::error::ExternalSecret ${namespace}/ghcr-auth did not materialise the Git/SOPS GHCR credential." + return 1 + fi +} # KSail embeds SOPS, so the deploy uses the same pinned toolchain as workload # reconciliation. Decrypt only the Docker config scalar and never emit it to @@ -54,22 +141,32 @@ chmod 600 "${docker_config}" if ! jq -e ' def non_empty_string: type == "string" and length > 0; (.auths["ghcr.io"] // {}) as $auth - | ( - (($auth.username | non_empty_string) - and ($auth.password | non_empty_string)) - or - (try ( + | ((($auth | has("username")) or ($auth | has("password")))) + as $explicit_present + | (($auth.username | non_empty_string) + and ($auth.password | non_empty_string)) as $explicit_valid + | ($auth | has("auth")) as $encoded_present + | (if $encoded_present then + try ( $auth.auth | @base64d | capture("^(?[^:]+):(?.+)$") - | ((.username | non_empty_string) - and (.password | non_empty_string)) - ) catch false) - ) + ) catch null + else null end) as $decoded + | (($decoded != null) + and ($decoded.username | non_empty_string) + and ($decoded.password | non_empty_string)) as $encoded_valid + | ((($explicit_present | not) or $explicit_valid) + and (($encoded_present | not) or $encoded_valid) + and ($explicit_valid or $encoded_valid) + and (((($explicit_present and $encoded_present) | not)) + or (($auth.username == $decoded.username) + and ($auth.password == $decoded.password)))) ' "${docker_config}" >/dev/null; then - echo "::error::The SOPS GHCR pull credential is not a valid Docker config with non-empty ghcr.io username/password or auth fields." + echo "::error::The SOPS GHCR pull credential is not a valid Docker config with non-empty, consistent ghcr.io username/password and auth fields." exit 1 fi +jq -S -c . "${docker_config}" > "${expected_normalized}" # Build curl's Basic-auth config without putting the credential in argv or # stdout. Support both Docker config representations used in this repository: @@ -143,9 +240,8 @@ if [[ "${check_only}" == "true" ]]; then exit 0 fi -# Merge only the Secret data field so KSail's ownership label and any unrelated -# metadata survive. The sensitive payload stays in the pipe/temp file and never -# appears in argv or logs. +# Merge only Secret data fields so ownership metadata survives. The sensitive +# payload stays in pipes/temp files and never appears in argv or logs. base64 < "${docker_config}" \ | tr -d '\r\n' \ | jq -Rs '{data: {".dockerconfigjson": .}}' \ @@ -158,4 +254,41 @@ kubectl \ --type=merge \ --patch-file="${patch_file}" -echo "✅ Refreshed the root Flux GHCR pull credential from Git/SOPS." +# A fresh DR cluster does not have variables-base or the ESO fan-out resources +# until its first Flux reconcile. In that case the current artifact creates the +# chain from the same SOPS value, so only the root bootstrap patch is needed. +if ! variables_base_name="$(kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + get secret variables-base \ + --ignore-not-found \ + -o name)"; then + echo "::error::Could not determine whether the GHCR fan-out exists; refusing to reconcile with an unverified tenant credential path." + exit 1 +fi +if [[ -z "${variables_base_name}" ]]; then + echo "✅ Refreshed root Flux GHCR auth; the first reconcile will create the downstream fan-out." + exit 0 +fi + +# Existing clusters must update the whole SOPS -> variables-base -> PushSecret +# -> OpenBao -> ExternalSecret chain before apps reconcile. Otherwise the root +# source recovers while tenant OCI/image pulls keep the revoked credential for +# up to their one-hour refresh interval. +jq '{data: {ghcr_dockerconfigjson: .data[".dockerconfigjson"]}}' \ + "${patch_file}" \ + > "${variables_patch_file}" +kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + patch secret variables-base \ + --type=merge \ + --patch-file="${variables_patch_file}" + +force_sync_resource pushsecret flux-system seed-ghcr +for namespace in "${FANOUT_NAMESPACES[@]}"; do + force_sync_resource externalsecret "${namespace}" ghcr-auth + verify_consumer_secret "${namespace}" +done + +echo "✅ Refreshed root Flux GHCR auth and synchronised every existing consumer from Git/SOPS." diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index cf082c1bb..f1ed4fbfc 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -29,9 +29,13 @@ def setUp(self) -> None: self.bin_dir.mkdir() self.decrypted_config = self.workspace / "decrypted-config.json" self.patch_capture = self.workspace / "patch.json" + self.variables_patch_capture = self.workspace / "variables-patch.json" self.kubectl_called = self.workspace / "kubectl-called" self.output_path_log = self.workspace / "ksail-output-path" self.registry_read_log = self.workspace / "registry-reads" + self.fanout_log = self.workspace / "fanout-log" + self.sync_state_dir = self.workspace / "sync-state" + self.sync_state_dir.mkdir() self._write_executable( "ksail", @@ -130,23 +134,92 @@ def setUp(self) -> None: set -euo pipefail arguments=" $* " [[ "$arguments" == *" --context admin@prod "* ]] - [[ "$arguments" == *" --namespace flux-system "* ]] - [[ "$arguments" == *" patch secret ksail-registry-credentials "* ]] - [[ "$arguments" == *" --type=merge "* ]] + namespace="" patch_file="" + previous="" for argument in "$@"; do + if [[ "$previous" == --namespace ]]; then + namespace="$argument" + fi case "$argument" in + --namespace=*) namespace="${argument#*=}" ;; --patch-file=*) patch_file="${argument#*=}" ;; esac + previous="$argument" done - test -n "$patch_file" + test -n "$namespace" touch "$KUBECTL_CALLED" - cp "$patch_file" "$PATCH_CAPTURE" - if [[ "${FAKE_KUBECTL_FAIL:-false}" == true ]]; then - echo 'cluster patch failed' >&2 - exit 43 + + if [[ "$arguments" == *" patch secret ksail-registry-credentials "* ]]; then + [[ "$arguments" == *" --type=merge "* ]] + test -n "$patch_file" + cp "$patch_file" "$PATCH_CAPTURE" + if [[ "${FAKE_KUBECTL_FAIL:-false}" == true ]]; then + echo 'cluster patch failed' >&2 + exit 43 + fi + echo 'secret/ksail-registry-credentials patched' + exit 0 + fi + + if [[ "$arguments" == *" get secret variables-base "* ]]; then + [[ "$arguments" == *" --ignore-not-found "* ]] + if [[ "${FAKE_VARIABLES_BASE_ABSENT:-false}" != true ]]; then + echo 'secret/variables-base' + fi + exit 0 + fi + + if [[ "$arguments" == *" patch secret variables-base "* ]]; then + [[ "$arguments" == *" --type=merge "* ]] + test -n "$patch_file" + cp "$patch_file" "$VARIABLES_PATCH_CAPTURE" + echo 'secret/variables-base patched' + exit 0 + fi + + kind="" + name="" + if [[ "$arguments" == *" pushsecret seed-ghcr "* ]]; then + kind=pushsecret + name=seed-ghcr + elif [[ "$arguments" == *" externalsecret ghcr-auth "* ]]; then + kind=externalsecret + name=ghcr-auth fi - echo 'secret/ksail-registry-credentials patched' + + if [[ -n "$kind" && "$arguments" == *" get $kind $name "* ]]; then + marker="$FAKE_SYNC_STATE_DIR/${kind}-${namespace}-${name}" + refresh_time=2026-07-13T00:00:00Z + if [[ -f "$marker" ]]; then + refresh_time=2026-07-13T00:00:01Z + fi + printf '{"status":{"refreshTime":"%s","conditions":[{"type":"Ready","status":"True"}]}}\n' "$refresh_time" + exit 0 + fi + + if [[ -n "$kind" && "$arguments" == *" annotate $kind $name "* ]]; then + resource="$kind/$namespace/$name" + printf '%s\n' "$resource" >> "$FANOUT_LOG" + if [[ "$resource" != "${FAKE_SYNC_STALL_RESOURCE:-disabled}" ]]; then + touch "$FAKE_SYNC_STATE_DIR/${kind}-${namespace}-${name}" + fi + echo "$kind/$name annotated" + exit 0 + fi + + if [[ "$arguments" == *" get secret ghcr-auth "* ]]; then + if [[ "$namespace" == "${FAKE_CONSUMER_MISMATCH_NAMESPACE:-disabled}" ]]; then + encoded=$(printf '%s' '{"auths":{}}' | base64 | tr -d '\r\n') + else + encoded=$(jq -r '.data.ghcr_dockerconfigjson' "$VARIABLES_PATCH_CAPTURE") + fi + jq -n --arg encoded "$encoded" '{data:{".dockerconfigjson":$encoded}}' + exit 0 + fi + + echo "unexpected kubectl invocation: $arguments" >&2 + exit 91 """, ) @@ -164,20 +237,29 @@ def _run_helper( self.decrypted_config.write_text(json.dumps(config), encoding="utf-8") for marker in ( self.patch_capture, + self.variables_patch_capture, self.kubectl_called, self.output_path_log, self.registry_read_log, + self.fanout_log, ): marker.unlink(missing_ok=True) + for marker in self.sync_state_dir.iterdir(): + marker.unlink() environment = os.environ.copy() environment.update( { "PATH": f"{self.bin_dir}:{environment['PATH']}", "FAKE_DECRYPTED_CONFIG": str(self.decrypted_config), "PATCH_CAPTURE": str(self.patch_capture), + "VARIABLES_PATCH_CAPTURE": str(self.variables_patch_capture), "KUBECTL_CALLED": str(self.kubectl_called), "KSAIL_OUTPUT_PATH_LOG": str(self.output_path_log), "REGISTRY_READ_LOG": str(self.registry_read_log), + "FANOUT_LOG": str(self.fanout_log), + "FAKE_SYNC_STATE_DIR": str(self.sync_state_dir), + "FLUX_GHCR_SYNC_ATTEMPTS": "2", + "FLUX_GHCR_SYNC_INTERVAL": "0", } ) environment.update(environment_overrides) @@ -201,7 +283,7 @@ def _valid_config() -> dict[str, object]: } } - def test_refreshes_only_the_root_secret_and_cleans_plaintext(self) -> None: + def test_refreshes_root_and_fanout_without_leaking_plaintext(self) -> None: config = self._valid_config() result = self._run_helper(config) @@ -211,6 +293,11 @@ def test_refreshes_only_the_root_secret_and_cleans_plaintext(self) -> None: encoded = patch["data"][".dockerconfigjson"] self.assertEqual(json.loads(base64.b64decode(encoded)), config) + variables_patch = json.loads( + self.variables_patch_capture.read_text(encoding="utf-8") + ) + variables_encoded = variables_patch["data"]["ghcr_dockerconfigjson"] + self.assertEqual(json.loads(base64.b64decode(variables_encoded)), config) temporary_config = Path(self.output_path_log.read_text(encoding="utf-8")) self.assertFalse(temporary_config.exists()) self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) @@ -222,6 +309,16 @@ def test_refreshes_only_the_root_secret_and_cleans_plaintext(self) -> None: "devantler-tech/ascoachingogvaner/manifests", "devantler-tech/wedding-app", "devantler-tech/ascoachingogvaner", + "devantler-tech/ksail", + ], + ) + self.assertEqual( + self.fanout_log.read_text(encoding="utf-8").splitlines(), + [ + "pushsecret/flux-system/seed-ghcr", + "externalsecret/wedding-app/ghcr-auth", + "externalsecret/ascoachingogvaner/ghcr-auth", + "externalsecret/kyverno/ghcr-auth", ], ) @@ -236,6 +333,17 @@ def test_accepts_standard_auth_only_docker_config(self) -> None: encoded = patch["data"][".dockerconfigjson"] self.assertEqual(json.loads(base64.b64decode(encoded)), config) + def test_accepts_matching_explicit_and_encoded_auth(self) -> None: + config = self._valid_config() + registry_auth = config["auths"]["ghcr.io"] + registry_auth["auth"] = base64.b64encode( + b"devantler:fixture-secret-token" + ).decode() + + result = self._run_helper(config) + + self.assertEqual(result.returncode, 0, result.stderr) + def test_check_only_preflights_without_patching(self) -> None: result = self._run_helper(self._valid_config(), ("--check-only",)) @@ -248,6 +356,17 @@ def test_missing_or_malformed_registry_auth_fails_closed(self) -> None: {"auths": {"ghcr.io": {"username": "devantler"}}}, {"auths": {"ghcr.io": {"username": "", "password": "token"}}}, {"auths": {"ghcr.io": {"auth": "not-base64"}}}, + { + "auths": { + "ghcr.io": { + "username": "devantler", + "password": "fixture-secret-token", + "auth": base64.b64encode( + b"devantler:different-token" + ).decode(), + } + } + }, {"auths": {}}, "not-a-docker-config", ] @@ -284,6 +403,40 @@ def test_cluster_patch_failure_is_not_hidden(self) -> None: self.assertEqual(result.returncode, 43) self.assertTrue(self.kubectl_called.exists()) + def test_fresh_cluster_without_variables_base_skips_existing_fanout(self) -> None: + result = self._run_helper( + self._valid_config(), FAKE_VARIABLES_BASE_ABSENT="true" + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.patch_capture.exists()) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + + def test_pushsecret_sync_failure_is_not_hidden(self) -> None: + result = self._run_helper( + self._valid_config(), + FAKE_SYNC_STALL_RESOURCE="pushsecret/flux-system/seed-ghcr", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertTrue(self.patch_capture.exists()) + self.assertTrue(self.variables_patch_capture.exists()) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + + def test_materialised_consumer_mismatch_is_not_hidden(self) -> None: + result = self._run_helper( + self._valid_config(), + FAKE_CONSUMER_MISMATCH_NAMESPACE="wedding-app", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "wedding-app/ghcr-auth did not materialise", + result.stdout + result.stderr, + ) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + class DeployActionOrderingTests(unittest.TestCase): """Keep credential refreshes on both sides of KSail-owned mutations.""" From 9dd1eec0eb60fc0f3a9f8e8dcf781b6b013817ec Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 01:14:18 +0200 Subject: [PATCH 06/16] fix(deploy): stage GHCR consumers before root auth --- .github/actions/deploy-prod/action.yml | 11 ++++---- AGENTS.md | 8 +++--- docs/dr/runbook.md | 14 +++++----- scripts/refresh-flux-ghcr-auth.sh | 27 ++++++++++++-------- scripts/tests/test_refresh_flux_ghcr_auth.py | 9 ++++++- 5 files changed, 42 insertions(+), 27 deletions(-) diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 47747f1fb..b54b6b402 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -295,10 +295,11 @@ runs: - name: 🔑 Reassert Flux GHCR pull credential after publish id: reassert_flux_ghcr_auth_after_push - # Revalidate the newly-published artifact, reassert root auth, and force - # the variables-base -> OpenBao -> tenant/Kyverno fan-out before apps - # reconcile. The initial preflight closed the mutable-tag failure window; - # this check proves the new tag and every private consumer are readable. + # Revalidate the newly-published artifact, force and verify the + # variables-base -> OpenBao -> tenant/Kyverno fan-out, and only then + # reassert root auth before apps reconcile. The initial preflight closed + # the mutable-tag failure window; this check proves the new tag and every + # private consumer are readable without exposing a partial fan-out. shell: bash run: ./scripts/refresh-flux-ghcr-auth.sh @@ -343,7 +344,7 @@ runs: - name: 🔑 Reassert Flux and consumer GHCR pull credential id: reassert_flux_ghcr_auth # cluster update can rewrite KSail's root Secret from GHCR_TOKEN when an - # unrelated configuration change triggers setup. Reassert Git/SOPS and + # unrelated configuration change triggers setup. Reassert Git/SOPS after # its synchronous consumer fan-out when that update fails part-way, but # only after the publish, signing, attestation, post-publish auth refresh, # and Flux reconcile all succeeded. A failed delivery must never expose a diff --git a/AGENTS.md b/AGENTS.md index 601012857..29e2eedd5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -146,7 +146,7 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 4. `ksail --config ksail.prod.yaml cluster create` (first run) or `cluster update` (subsequent runs) provisions / reconciles the Hetzner servers, Talos, CCM, and CSI. 5. `scripts/refresh-flux-ghcr-auth.sh --check-only` decrypts only the Git/SOPS pull credential and performs real OCI manifest reads for all six private consumers (the Platform and tenant manifest artifacts, both tenant application images, and the KSail package used by Kyverno signature verification) before a mutable `latest` tag is published; the DR workflow also runs it before creating infrastructure. It does not mutate the cluster. 6. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. -7. The bridge revalidates the newly-published artifact, reasserts root auth, updates `variables-base`, and force-syncs the PushSecret plus tenant/Kyverno ExternalSecrets before reconciliation. +7. The bridge revalidates the newly-published artifact, updates `variables-base`, force-syncs and verifies the PushSecret plus tenant/Kyverno ExternalSecrets, and only then reasserts root auth before reconciliation. A fresh cluster without the fan-out resources repairs root auth first so Flux can bootstrap them. 8. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. **Key differences from local:** @@ -165,7 +165,7 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o - **`ci.yaml`** — runs on `pull_request` (static manifest validation + Kubescape scan, no cluster) and `merge_group` (deploys prod via the Hetzner provider). Concurrency is shared with `cd.yaml` so a manual deploy and a merge-queue deploy can never run against the prod cluster at the same time. - **`cd.yaml`** — runs on `workflow_dispatch` (manual). Deploys to the production Hetzner cluster using `ksail --config ksail.prod.yaml`. Covers direct pushes to `main`, which bypass the merge queue and so are not deployed by `ci.yaml`. -- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (preflight root GHCR auth → push → cosign-sign → attest SBOM + SLSA provenance → revalidate/patch root auth → Flux reconcile → Talos `cluster update` → final reassert), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. +- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (preflight root GHCR auth → push → cosign-sign → attest SBOM + SLSA provenance → revalidate/stage consumers/patch root auth → Flux reconcile → Talos `cluster update` → final reassert), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. **Required GitHub Secrets:** @@ -178,9 +178,11 @@ The authoritative **Flux and tenant** GHCR pull credential is `k8s/bases/bootstrap/secret.enc.yaml`. The deploy bridge refreshes `flux-system/ksail-registry-credentials` from that value before Flux must fetch the artifact and reasserts it after `cluster update` in case KSail rewrites its -managed Secret. On existing clusters the bridge also updates `variables-base`, +managed Secret. On existing clusters the bridge first updates `variables-base`, force-syncs `seed-ghcr` into OpenBao, force-syncs the tenant/Kyverno ExternalSecrets, and verifies their materialised `ghcr-auth` payloads before +switching root Flux auth. A fresh cluster repairs root auth first because the +fan-out resources do not exist until Flux can fetch the initial artifact. apps reconcile. A direct credential commit to `main` still needs a manual `CD` workflow dispatch because direct pushes bypass the merge-queue deploy. Talos node registry auth still derives from `GHCR_TOKEN`; consolidating that diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index 73087e406..4cf4f39e4 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -511,14 +511,14 @@ The production deploy closes the bootstrap loop in this order: `latest` tag (and before infrastructure creation during DR), and does not mutate the cluster. 2. Push and sign the new artifact with `GHCR_TOKEN`, then revalidate the SOPS - credential against the newly-published artifact. Patch - `flux-system/ksail-registry-credentials` and, on an existing cluster, - `variables-base`; force-sync `seed-ghcr` into OpenBao; force-sync the - tenant/Kyverno ExternalSecrets; and verify every materialised `ghcr-auth` - payload matches Git/SOPS. + credential against the newly-published artifact. On an existing cluster, + patch `variables-base`; force-sync `seed-ghcr` into OpenBao; force-sync the + tenant/Kyverno ExternalSecrets; verify every materialised `ghcr-auth` + payload matches Git/SOPS; and only then patch + `flux-system/ksail-registry-credentials`. 3. Reconcile Flux only after that synchronous fan-out succeeds. A fresh DR - cluster has no fan-out resources yet, so its first reconcile creates the - entire chain directly from the same current artifact. + cluster has no fan-out resources yet, so the bridge patches root auth first + and its first reconcile creates the entire chain from the same artifact. 4. Re-run the bridge after `cluster update`, because KSail can rewrite its managed root Secret when another cluster setting changes; the bridge also re-verifies the downstream fan-out. diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index 1de78110e..bb47730e7 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -247,12 +247,14 @@ base64 < "${docker_config}" \ | jq -Rs '{data: {".dockerconfigjson": .}}' \ > "${patch_file}" -kubectl \ - --context "${KUBE_CONTEXT}" \ - --namespace flux-system \ - patch secret ksail-registry-credentials \ - --type=merge \ - --patch-file="${patch_file}" +patch_root_secret() { + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + patch secret ksail-registry-credentials \ + --type=merge \ + --patch-file="${patch_file}" +} # A fresh DR cluster does not have variables-base or the ESO fan-out resources # until its first Flux reconcile. In that case the current artifact creates the @@ -267,14 +269,15 @@ if ! variables_base_name="$(kubectl \ exit 1 fi if [[ -z "${variables_base_name}" ]]; then + patch_root_secret echo "✅ Refreshed root Flux GHCR auth; the first reconcile will create the downstream fan-out." exit 0 fi -# Existing clusters must update the whole SOPS -> variables-base -> PushSecret -# -> OpenBao -> ExternalSecret chain before apps reconcile. Otherwise the root -# source recovers while tenant OCI/image pulls keep the revoked credential for -# up to their one-hour refresh interval. +# Existing clusters must update and verify the whole SOPS -> variables-base -> +# PushSecret -> OpenBao -> ExternalSecret chain before switching root Flux auth. +# Otherwise a failed fan-out can still let normal root-source polling apply the +# newly-published artifact while tenant OCI/image pulls keep stale credentials. jq '{data: {ghcr_dockerconfigjson: .data[".dockerconfigjson"]}}' \ "${patch_file}" \ > "${variables_patch_file}" @@ -291,4 +294,6 @@ for namespace in "${FANOUT_NAMESPACES[@]}"; do verify_consumer_secret "${namespace}" done -echo "✅ Refreshed root Flux GHCR auth and synchronised every existing consumer from Git/SOPS." +patch_root_secret + +echo "✅ Synchronised every existing consumer and refreshed root Flux GHCR auth from Git/SOPS." diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index f1ed4fbfc..1e97b43cf 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -420,7 +420,10 @@ def test_pushsecret_sync_failure_is_not_hidden(self) -> None: ) self.assertNotEqual(result.returncode, 0) - self.assertTrue(self.patch_capture.exists()) + self.assertFalse( + self.patch_capture.exists(), + "root Flux auth must remain unchanged until fan-out verifies", + ) self.assertTrue(self.variables_patch_capture.exists()) self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) @@ -435,6 +438,10 @@ def test_materialised_consumer_mismatch_is_not_hidden(self) -> None: "wedding-app/ghcr-auth did not materialise", result.stdout + result.stderr, ) + self.assertFalse( + self.patch_capture.exists(), + "root Flux auth must remain unchanged until consumers match", + ) self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) From 00141e537d3e9ce286fb5114b3d277fcdc9a0d7c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 01:37:25 +0200 Subject: [PATCH 07/16] fix(deploy): close GHCR credential race windows --- .github/actions/deploy-prod/action.yml | 35 ++- .github/workflows/dr-rebuild.yaml | 25 ++- AGENTS.md | 14 +- docs/dr/runbook.md | 30 +-- .../vault-seed/push-secret-seed-ghcr.yaml | 7 +- scripts/refresh-flux-ghcr-auth.sh | 120 ++++++++-- scripts/tests/test_refresh_flux_ghcr_auth.py | 209 +++++++++++++++--- 7 files changed, 345 insertions(+), 95 deletions(-) diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index b54b6b402..2b021e944 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -126,14 +126,14 @@ runs: echo "::error::Prod cluster (context admin@prod) is unreachable with the restored kubeconfig (last kubectl error: ${last_err}). KUBE_CONFIG (and likely TALOS_CONFIG) are probably stale after a cluster rebuild — refresh both and update the 'prod' environment secrets. See docs/dr/runbook.md → 'Refresh CI deploy credentials'." exit 1 - - name: 🔎 Preflight Flux GHCR pull credential - id: preflight_flux_ghcr_auth - # Perform a real OCI read for all six packages served by this credential - # BEFORE publishing latest. Do not patch yet: the matching artifact has - # not been published. This prevents a bad SOPS rotation from being - # discovered only after the mutable tag changes. + - name: 🔑 Stage Flux and consumer GHCR pull credential + id: stage_flux_ghcr_auth + # Validate all seven packages, then synchronously stage and verify the whole + # pull-credential fan-out BEFORE publishing latest. If the mutable tag is + # observed by Flux immediately after push, every consumer already has the + # same known-readable credential. shell: bash - run: ./scripts/refresh-flux-ghcr-auth.sh --check-only + run: ./scripts/refresh-flux-ghcr-auth.sh # Publish the GitOps manifests (push → sign → attest → reconcile) BEFORE # syncing Talos machine config (the "🔄 Update cluster" step runs LAST). @@ -293,15 +293,12 @@ runs: push-to-registry: true create-storage-record: false - - name: 🔑 Reassert Flux GHCR pull credential after publish - id: reassert_flux_ghcr_auth_after_push - # Revalidate the newly-published artifact, force and verify the - # variables-base -> OpenBao -> tenant/Kyverno fan-out, and only then - # reassert root auth before apps reconcile. The initial preflight closed - # the mutable-tag failure window; this check proves the new tag and every - # private consumer are readable without exposing a partial fan-out. + - name: 🔎 Verify Flux GHCR pull credential after publish + id: verify_flux_ghcr_auth_after_push + # Revalidate the newly-published artifact before apps reconcile. The + # complete consumer fan-out was staged before the mutable tag moved. shell: bash - run: ./scripts/refresh-flux-ghcr-auth.sh + run: ./scripts/refresh-flux-ghcr-auth.sh --check-only - name: 🔁 Trigger Flux reconciliation id: reconcile @@ -346,12 +343,12 @@ runs: # cluster update can rewrite KSail's root Secret from GHCR_TOKEN when an # unrelated configuration change triggers setup. Reassert Git/SOPS after # its synchronous consumer fan-out when that update fails part-way, but - # only after the publish, signing, attestation, post-publish auth refresh, - # and Flux reconcile all succeeded. A failed delivery must never expose a - # partial latest tag to Flux's normal polling loop. + # only after the publish, signing, attestation, post-publish auth check, + # and Flux reconcile all succeeded. Never mutate auth as cleanup for a + # failed delivery. if: >- always() && - steps.reassert_flux_ghcr_auth_after_push.outcome == 'success' && + steps.verify_flux_ghcr_auth_after_push.outcome == 'success' && steps.reconcile.outcome == 'success' shell: bash run: ./scripts/refresh-flux-ghcr-auth.sh diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index a9bf94d68..5a7918154 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -114,6 +114,14 @@ jobs: # renders an empty key and Talos rejects the whole machine config. WG_SERVER_PRIVATE_KEY: ${{ secrets.WG_SERVER_PRIVATE_KEY }} + - name: 🔑 Stage Flux and consumer GHCR pull credential + id: stage_flux_ghcr_auth + # cluster create bootstraps with GHCR_TOKEN for KSail compatibility. + # Make Git/SOPS authoritative before the mutable tag is published. A + # partial bootstrap may have variables-base but no ESO resources yet; + # the bridge safely repairs root auth so Flux can create them. + run: ./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout + - name: 📦 Push manifests to GHCR run: ksail --config ksail.prod.yaml workload push env: @@ -121,11 +129,11 @@ jobs: GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - - name: 🔑 Refresh Flux GHCR pull credential - # cluster create bootstraps with GHCR_TOKEN for KSail compatibility, - # then this bridge makes the Git/SOPS pull credential authoritative - # before Flux performs its first artifact fetch. - run: ./scripts/refresh-flux-ghcr-auth.sh + - name: 🔎 Verify Flux GHCR pull credential after publish + id: verify_flux_ghcr_auth_after_push + # Prove the newly-published artifact is readable before reconciliation; + # every available consumer was already staged before push. + run: ./scripts/refresh-flux-ghcr-auth.sh --check-only - name: 🔁 Trigger Flux reconciliation run: ksail --config ksail.prod.yaml workload reconcile @@ -143,6 +151,13 @@ jobs: done echo "✅ All Flux Kustomizations Ready — fresh platform converged." + - name: 🔑 Verify completed GHCR credential fan-out + id: verify_flux_ghcr_fanout + # Bootstrap mode may repair only root auth when ESO is not installed + # yet. After Flux converges, require the complete PushSecret / + # ExternalSecret transaction and exact consumer payload verification. + run: ./scripts/refresh-flux-ghcr-auth.sh + - name: 💾 Velero resource restore (newest Completed backup) if: ${{ inputs.restore }} run: | diff --git a/AGENTS.md b/AGENTS.md index 29e2eedd5..c779fcd35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,10 +144,11 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 2. The `deploy-prod` composite action (shared by both paths) uses `ksail --config ksail.prod.yaml` to target the committed prod config. 3. `ksail.prod.yaml` has `kustomizationFile: clusters/prod`, so KSail/Flux use `k8s/clusters/prod/kustomization.yaml` as the entry point — no root `k8s/kustomization.yaml` or file rewriting is needed. 4. `ksail --config ksail.prod.yaml cluster create` (first run) or `cluster update` (subsequent runs) provisions / reconciles the Hetzner servers, Talos, CCM, and CSI. -5. `scripts/refresh-flux-ghcr-auth.sh --check-only` decrypts only the Git/SOPS pull credential and performs real OCI manifest reads for all six private consumers (the Platform and tenant manifest artifacts, both tenant application images, and the KSail package used by Kyverno signature verification) before a mutable `latest` tag is published; the DR workflow also runs it before creating infrastructure. It does not mutate the cluster. +5. The bridge decrypts only the Git/SOPS pull credential, performs real OCI manifest reads for all seven private consumers (the Platform and tenant manifest artifacts, both tenant application images, and the KSail plus provider-upjet-unifi packages used by Kyverno verification), updates `variables-base`, force-syncs and verifies the PushSecret plus tenant/Kyverno ExternalSecrets, and only then reasserts root auth — all before a mutable `latest` tag is published. The DR workflow first runs `--check-only` before creating infrastructure, then uses explicit `--allow-incomplete-fanout` bootstrap mode after cluster creation and requires a full bridge pass after Flux converges. 6. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. -7. The bridge revalidates the newly-published artifact, updates `variables-base`, force-syncs and verifies the PushSecret plus tenant/Kyverno ExternalSecrets, and only then reasserts root auth before reconciliation. A fresh cluster without the fan-out resources repairs root auth first so Flux can bootstrap them. +7. `scripts/refresh-flux-ghcr-auth.sh --check-only` revalidates the newly-published artifact without mutating the cluster. 8. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. +9. After `cluster update`, the full bridge reasserts the Git/SOPS credential in case KSail rewrote its managed root Secret. **Key differences from local:** @@ -165,7 +166,7 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o - **`ci.yaml`** — runs on `pull_request` (static manifest validation + Kubescape scan, no cluster) and `merge_group` (deploys prod via the Hetzner provider). Concurrency is shared with `cd.yaml` so a manual deploy and a merge-queue deploy can never run against the prod cluster at the same time. - **`cd.yaml`** — runs on `workflow_dispatch` (manual). Deploys to the production Hetzner cluster using `ksail --config ksail.prod.yaml`. Covers direct pushes to `main`, which bypass the merge queue and so are not deployed by `ci.yaml`. -- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (preflight root GHCR auth → push → cosign-sign → attest SBOM + SLSA provenance → revalidate/stage consumers/patch root auth → Flux reconcile → Talos `cluster update` → final reassert), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. +- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (stage/verify all GHCR pull consumers → push → cosign-sign → attest SBOM + SLSA provenance → revalidate published artifact → Flux reconcile → Talos `cluster update` → final reassert), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. **Required GitHub Secrets:** @@ -178,11 +179,12 @@ The authoritative **Flux and tenant** GHCR pull credential is `k8s/bases/bootstrap/secret.enc.yaml`. The deploy bridge refreshes `flux-system/ksail-registry-credentials` from that value before Flux must fetch the artifact and reasserts it after `cluster update` in case KSail rewrites its -managed Secret. On existing clusters the bridge first updates `variables-base`, +managed Secret. Before publish on existing clusters, the bridge updates `variables-base`, force-syncs `seed-ghcr` into OpenBao, force-syncs the tenant/Kyverno ExternalSecrets, and verifies their materialised `ghcr-auth` payloads before -switching root Flux auth. A fresh cluster repairs root auth first because the -fan-out resources do not exist until Flux can fetch the initial artifact. +switching root Flux auth. Only explicit DR bootstrap mode may repair root auth +after staging `variables-base` while the fan-out is incomplete; DR must run the +full verifier after Flux converges. apps reconcile. A direct credential commit to `main` still needs a manual `CD` workflow dispatch because direct pushes bypass the merge-queue deploy. Talos node registry auth still derives from `GHCR_TOKEN`; consolidating that diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index 4cf4f39e4..7ab242a16 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -505,20 +505,22 @@ single-source consolidation stays tracked by #2613. The production deploy closes the bootstrap loop in this order: 1. Decrypt only `ghcr_dockerconfigjson` and perform real OCI manifest reads for - all six private consumers: Platform manifests, both tenant manifest - artifacts, both tenant application images, and the KSail package used by - Kyverno signature verification. This happens before changing a mutable - `latest` tag (and before infrastructure creation during DR), and does not - mutate the cluster. -2. Push and sign the new artifact with `GHCR_TOKEN`, then revalidate the SOPS - credential against the newly-published artifact. On an existing cluster, - patch `variables-base`; force-sync `seed-ghcr` into OpenBao; force-sync the - tenant/Kyverno ExternalSecrets; verify every materialised `ghcr-auth` - payload matches Git/SOPS; and only then patch - `flux-system/ksail-registry-credentials`. -3. Reconcile Flux only after that synchronous fan-out succeeds. A fresh DR - cluster has no fan-out resources yet, so the bridge patches root auth first - and its first reconcile creates the entire chain from the same artifact. + all seven private consumers: Platform manifests, both tenant manifest + artifacts, both tenant application images, and the KSail plus + provider-upjet-unifi packages used by Kyverno verification. During DR, a + read-only `--check-only` pass happens before infrastructure creation. +2. Before changing mutable `latest`, patch `variables-base`; force-sync + `seed-ghcr` into OpenBao; force-sync the tenant/Kyverno ExternalSecrets; + verify every materialised `ghcr-auth` payload matches Git/SOPS; and only then + patch `flux-system/ksail-registry-credentials`. A fresh or partial DR + bootstrap may have no ESO CRDs/resources yet; explicit + `--allow-incomplete-fanout` mode stages `variables-base` and repairs root auth + so the first reconcile can create the chain. Normal mode fails closed on any + missing fan-out resource. +3. Push and sign the artifact with `GHCR_TOKEN`, revalidate the newly-published + artifact with `--check-only`, and only then explicitly reconcile Flux. DR + runs the full bridge again after every Flux Kustomization is Ready, proving + that bootstrap mode completed the entire fan-out. 4. Re-run the bridge after `cluster update`, because KSail can rewrite its managed root Secret when another cluster setting changes; the bridge also re-verifies the downstream fan-out. diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml index c15587603..df1ec292e 100644 --- a/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml +++ b/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml @@ -16,9 +16,10 @@ # This one is deliberately SOPS-seeded, NOT user-fed, even though it is an # upstream token: the verify-image-signatures ClusterPolicy needs the kyverno # `ghcr-auth` Secret to fetch signature manifests of PRIVATE first-party -# images (ksail-operator runs in EVERY cluster, including ephemeral CI ones -# where no operator exists to feed the vault) — without it Kyverno denies -# those pods at admission. That makes it bootstrap machinery. +# images. KSail and provider-upjet packages run in managed clusters, including +# ephemeral CI ones where no operator exists to feed the vault; without this +# Secret, Kyverno denies those pods at admission. That makes it bootstrap +# machinery. apiVersion: external-secrets.io/v1alpha1 kind: PushSecret metadata: diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index bb47730e7..d83750b6e 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -8,29 +8,34 @@ set -euo pipefail check_only=false +allow_incomplete_fanout=false if (($# > 1)); then - echo "Usage: $0 [--check-only]" >&2 + echo "Usage: $0 [--check-only|--allow-incomplete-fanout]" >&2 exit 64 fi if (($# == 1)); then - if [[ "$1" != "--check-only" ]]; then - echo "Usage: $0 [--check-only]" >&2 - exit 64 - fi - check_only=true + case "$1" in + --check-only) check_only=true ;; + --allow-incomplete-fanout) allow_incomplete_fanout=true ;; + *) + echo "Usage: $0 [--check-only|--allow-incomplete-fanout]" >&2 + exit 64 + ;; + esac fi readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" readonly KUBE_CONTEXT="${KUBE_CONTEXT:-admin@prod}" readonly SYNC_ATTEMPTS="${FLUX_GHCR_SYNC_ATTEMPTS:-60}" readonly SYNC_INTERVAL="${FLUX_GHCR_SYNC_INTERVAL:-2}" -readonly -a REQUIRED_PULL_REPOSITORIES=( - "devantler-tech/platform/manifests" - "devantler-tech/wedding-app/manifests" - "devantler-tech/ascoachingogvaner/manifests" - "devantler-tech/wedding-app" - "devantler-tech/ascoachingogvaner" - "devantler-tech/ksail" +readonly -a REQUIRED_PULL_TARGETS=( + "devantler-tech/platform/manifests:latest" + "devantler-tech/wedding-app/manifests:latest" + "devantler-tech/ascoachingogvaner/manifests:latest" + "devantler-tech/wedding-app:latest" + "devantler-tech/ascoachingogvaner:latest" + "devantler-tech/ksail:latest" + "devantler-tech/provider-upjet-unifi:v0.1.0" ) readonly -a FANOUT_NAMESPACES=( "wedding-app" @@ -56,14 +61,17 @@ token_response="${work_dir}/token.json" patch_file="${work_dir}/patch.json" variables_patch_file="${work_dir}/variables-patch.json" expected_normalized="${work_dir}/expected-normalized.json" +fanout_api_resources="${work_dir}/fanout-api-resources.txt" force_sync_resource() { local kind="$1" local namespace="$2" local name="$3" local before_file="${work_dir}/${kind}-${namespace}-${name}-before.json" + local annotated_file="${work_dir}/${kind}-${namespace}-${name}-annotated.json" local current_file="${work_dir}/${kind}-${namespace}-${name}-current.json" local before_refresh + local annotated_resource_version local attempt local stamp @@ -82,7 +90,10 @@ force_sync_resource() { annotate "${kind}" "${name}" \ "force-sync=${stamp}" \ --overwrite \ - >/dev/null + -o json \ + > "${annotated_file}" + annotated_resource_version="$(jq -er '.metadata.resourceVersion' \ + "${annotated_file}")" for ((attempt = 1; attempt <= SYNC_ATTEMPTS; attempt++)); do kubectl \ @@ -91,9 +102,13 @@ force_sync_resource() { get "${kind}" "${name}" \ -o json \ > "${current_file}" - if jq -e --arg before "${before_refresh}" ' + if jq -e \ + --arg before "${before_refresh}" \ + --arg annotated_resource_version "${annotated_resource_version}" ' (.status.refreshTime // "") as $refresh - | ($refresh != "" and $refresh != $before) + | (($refresh != "" and $refresh != $before) + or ((.metadata.resourceVersion // "") != "" + and .metadata.resourceVersion != $annotated_resource_version)) and any(.status.conditions[]?; .type == "Ready" and .status == "True") ' "${current_file}" >/dev/null; then @@ -190,7 +205,9 @@ chmod 600 "${basic_curl_config}" # then perform a real registry manifest GET for every package. Both credentials # stay in mode-0600 files. --disable must remain curl's first argument so an # ambient ~/.curlrc cannot enable tracing, add URLs, or otherwise expose auth. -for repository in "${REQUIRED_PULL_REPOSITORIES[@]}"; do +for target in "${REQUIRED_PULL_TARGETS[@]}"; do + repository="${target%:*}" + reference="${target##*:}" if ! http_status="$(curl --disable \ --config "${basic_curl_config}" \ --silent \ @@ -225,12 +242,12 @@ for repository in "${REQUIRED_PULL_REPOSITORIES[@]}"; do --output /dev/null \ --write-out '%{http_code}' \ --header 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json' \ - "https://ghcr.io/v2/${repository}/manifests/latest")"; then - echo "::error::Could not read the GHCR manifest for ${repository}; the root Flux Secret was not changed." + "https://ghcr.io/v2/${repository}/manifests/${reference}")"; then + echo "::error::Could not read the GHCR manifest for ${target}; the root Flux Secret was not changed." exit 1 fi if [[ "${http_status}" != "200" ]]; then - echo "::error::The SOPS GHCR pull credential cannot read ${repository}:latest (GHCR HTTP ${http_status}); the root Flux Secret was not changed." + echo "::error::The SOPS GHCR pull credential cannot read ${target} (GHCR HTTP ${http_status}); the root Flux Secret was not changed." exit 1 fi done @@ -269,6 +286,10 @@ if ! variables_base_name="$(kubectl \ exit 1 fi if [[ -z "${variables_base_name}" ]]; then + if [[ "${allow_incomplete_fanout}" != "true" ]]; then + echo "::error::The GHCR fan-out is not initialized; root Flux auth was not changed. Use --allow-incomplete-fanout only during the DR bootstrap, then run the full verifier after reconciliation." + exit 1 + fi patch_root_secret echo "✅ Refreshed root Flux GHCR auth; the first reconcile will create the downstream fan-out." exit 0 @@ -288,6 +309,65 @@ kubectl \ --type=merge \ --patch-file="${variables_patch_file}" +# A partially-bootstrapped DR cluster can already have variables-base while ESO +# CRDs or individual fan-out objects do not exist yet. That state still needs +# root auth so Flux can fetch the artifact that completes the chain. Distinguish +# an absent API/resource from a failed lookup, and never force-sync a partial set. +if ! kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + api-resources \ + --api-group=external-secrets.io \ + -o name \ + > "${fanout_api_resources}"; then + echo "::error::Could not inspect the External Secrets API; refusing to change root Flux auth." + exit 1 +fi + +fanout_complete=true +if ! grep -qx 'pushsecrets.external-secrets.io' "${fanout_api_resources}" \ + || ! grep -qx 'externalsecrets.external-secrets.io' "${fanout_api_resources}"; then + fanout_complete=false +else + if ! pushsecret_name="$(kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + get pushsecret seed-ghcr \ + --ignore-not-found \ + -o name)"; then + echo "::error::Could not determine whether PushSecret flux-system/seed-ghcr exists; refusing to change root Flux auth." + exit 1 + fi + if [[ -z "${pushsecret_name}" ]]; then + fanout_complete=false + fi + + for namespace in "${FANOUT_NAMESPACES[@]}"; do + if ! externalsecret_name="$(kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + get externalsecret ghcr-auth \ + --ignore-not-found \ + -o name)"; then + echo "::error::Could not determine whether ExternalSecret ${namespace}/ghcr-auth exists; refusing to change root Flux auth." + exit 1 + fi + if [[ -z "${externalsecret_name}" ]]; then + fanout_complete=false + fi + done +fi + +if [[ "${fanout_complete}" != "true" ]]; then + if [[ "${allow_incomplete_fanout}" != "true" ]]; then + echo "::error::The GHCR fan-out is incomplete; root Flux auth was not changed. Use --allow-incomplete-fanout only during the DR bootstrap, then run the full verifier after reconciliation." + exit 1 + fi + patch_root_secret + echo "✅ Staged the Git/SOPS credential and refreshed root Flux auth; the first reconcile will complete the missing downstream fan-out." + exit 0 +fi + force_sync_resource pushsecret flux-system seed-ghcr for namespace in "${FANOUT_NAMESPACES[@]}"; do force_sync_resource externalsecret "${namespace}" ghcr-auth diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index 1e97b43cf..c4293311b 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -16,6 +16,15 @@ HELPER = ROOT / "scripts" / "refresh-flux-ghcr-auth.sh" ACTION = ROOT / ".github" / "actions" / "deploy-prod" / "action.yml" DR_REBUILD = ROOT / ".github" / "workflows" / "dr-rebuild.yaml" +PROVIDER_UPJET_UNIFI = ( + ROOT + / "k8s" + / "providers" + / "hetzner" + / "infrastructure" + / "crossplane" + / "provider-upjet-unifi.yaml" +) class RefreshFluxGhcrAuthTests(unittest.TestCase): @@ -115,11 +124,13 @@ def setUp(self) -> None: exit 0 fi - repository="${url#https://ghcr.io/v2/}" - repository="${repository%/manifests/latest}" - test "$repository" != "$url" + manifest_path="${url#https://ghcr.io/v2/}" + repository="${manifest_path%/manifests/*}" + reference="${manifest_path##*/manifests/}" + test "$repository" != "$manifest_path" + test "$reference" != "$manifest_path" grep -q 'Authorization: Bearer fixture-registry-token' "$config" - printf '%s\n' "$repository" >> "$REGISTRY_READ_LOG" + printf '%s:%s\n' "$repository" "$reference" >> "$REGISTRY_READ_LOG" if [[ "$repository" == "${FAKE_CURL_DENY_REPOSITORY:-disabled}" ]]; then printf '403' else @@ -150,6 +161,16 @@ def setUp(self) -> None: test -n "$namespace" touch "$KUBECTL_CALLED" + if [[ "$arguments" == *" api-resources "* ]]; then + [[ "$arguments" == *" --api-group=external-secrets.io "* ]] + if [[ "${FAKE_FANOUT_CRDS_ABSENT:-false}" != true ]]; then + printf '%s\n' \ + 'externalsecrets.external-secrets.io' \ + 'pushsecrets.external-secrets.io' + fi + exit 0 + fi + if [[ "$arguments" == *" patch secret ksail-registry-credentials "* ]]; then [[ "$arguments" == *" --type=merge "* ]] test -n "$patch_file" @@ -188,23 +209,48 @@ def setUp(self) -> None: name=ghcr-auth fi + if [[ -n "$kind" && "$arguments" == *" get $kind $name "* \ + && "$arguments" == *" --ignore-not-found "* ]]; then + resource="$kind/$namespace/$name" + if [[ "$resource" != "${FAKE_MISSING_FANOUT_RESOURCE:-disabled}" ]]; then + echo "$kind/$name" + fi + exit 0 + fi + + if [[ -n "$kind" \ + && "$kind/$namespace/$name" == "${FAKE_MISSING_FANOUT_RESOURCE:-disabled}" ]]; then + echo "$kind/$name not found" >&2 + exit 44 + fi + if [[ -n "$kind" && "$arguments" == *" get $kind $name "* ]]; then marker="$FAKE_SYNC_STATE_DIR/${kind}-${namespace}-${name}" + annotated_marker="${marker}-annotated" refresh_time=2026-07-13T00:00:00Z - if [[ -f "$marker" ]]; then + resource_version=1 + if [[ -f "$annotated_marker" ]]; then + resource_version=2 + fi + if [[ -f "$marker" && "${FAKE_SYNC_SAME_REFRESH_TIME:-false}" != true ]]; then refresh_time=2026-07-13T00:00:01Z fi - printf '{"status":{"refreshTime":"%s","conditions":[{"type":"Ready","status":"True"}]}}\n' "$refresh_time" + if [[ -f "$marker" ]]; then + resource_version=3 + fi + printf '{"metadata":{"resourceVersion":"%s"},"status":{"refreshTime":"%s","conditions":[{"type":"Ready","status":"True"}]}}\n' "$resource_version" "$refresh_time" exit 0 fi if [[ -n "$kind" && "$arguments" == *" annotate $kind $name "* ]]; then resource="$kind/$namespace/$name" printf '%s\n' "$resource" >> "$FANOUT_LOG" + marker="$FAKE_SYNC_STATE_DIR/${kind}-${namespace}-${name}" + touch "${marker}-annotated" if [[ "$resource" != "${FAKE_SYNC_STALL_RESOURCE:-disabled}" ]]; then - touch "$FAKE_SYNC_STATE_DIR/${kind}-${namespace}-${name}" + touch "$marker" fi - echo "$kind/$name annotated" + printf '{"metadata":{"resourceVersion":"2"}}\n' exit 0 fi @@ -304,12 +350,13 @@ def test_refreshes_root_and_fanout_without_leaking_plaintext(self) -> None: self.assertEqual( self.registry_read_log.read_text(encoding="utf-8").splitlines(), [ - "devantler-tech/platform/manifests", - "devantler-tech/wedding-app/manifests", - "devantler-tech/ascoachingogvaner/manifests", - "devantler-tech/wedding-app", - "devantler-tech/ascoachingogvaner", - "devantler-tech/ksail", + "devantler-tech/platform/manifests:latest", + "devantler-tech/wedding-app/manifests:latest", + "devantler-tech/ascoachingogvaner/manifests:latest", + "devantler-tech/wedding-app:latest", + "devantler-tech/ascoachingogvaner:latest", + "devantler-tech/ksail:latest", + "devantler-tech/provider-upjet-unifi:v0.1.0", ], ) self.assertEqual( @@ -405,7 +452,9 @@ def test_cluster_patch_failure_is_not_hidden(self) -> None: def test_fresh_cluster_without_variables_base_skips_existing_fanout(self) -> None: result = self._run_helper( - self._valid_config(), FAKE_VARIABLES_BASE_ABSENT="true" + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_VARIABLES_BASE_ABSENT="true", ) self.assertEqual(result.returncode, 0, result.stderr) @@ -413,6 +462,60 @@ def test_fresh_cluster_without_variables_base_skips_existing_fanout(self) -> Non self.assertFalse(self.variables_patch_capture.exists()) self.assertFalse(self.fanout_log.exists()) + def test_missing_variables_base_fails_closed_without_bootstrap_mode(self) -> None: + result = self._run_helper( + self._valid_config(), + FAKE_VARIABLES_BASE_ABSENT="true", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.patch_capture.exists()) + + def test_partial_bootstrap_repairs_root_without_forcing_missing_fanout( + self, + ) -> None: + missing_resources = [ + "pushsecret/flux-system/seed-ghcr", + "externalsecret/wedding-app/ghcr-auth", + "externalsecret/ascoachingogvaner/ghcr-auth", + "externalsecret/kyverno/ghcr-auth", + ] + for resource in missing_resources: + with self.subTest(resource=resource): + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_MISSING_FANOUT_RESOURCE=resource, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.variables_patch_capture.exists()) + self.assertTrue(self.patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + self.assertIn("first reconcile will complete", result.stdout) + + def test_partial_fanout_fails_closed_without_bootstrap_mode(self) -> None: + result = self._run_helper( + self._valid_config(), + FAKE_MISSING_FANOUT_RESOURCE="externalsecret/kyverno/ghcr-auth", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + + def test_partial_bootstrap_without_eso_crds_repairs_root(self) -> None: + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_FANOUT_CRDS_ABSENT="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.variables_patch_capture.exists()) + self.assertTrue(self.patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + def test_pushsecret_sync_failure_is_not_hidden(self) -> None: result = self._run_helper( self._valid_config(), @@ -427,6 +530,15 @@ def test_pushsecret_sync_failure_is_not_hidden(self) -> None: self.assertTrue(self.variables_patch_capture.exists()) self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + def test_same_second_sync_accepts_controller_resource_version_edge(self) -> None: + result = self._run_helper( + self._valid_config(), + FAKE_SYNC_SAME_REFRESH_TIME="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.patch_capture.exists()) + def test_materialised_consumer_mismatch_is_not_hidden(self) -> None: result = self._run_helper( self._valid_config(), @@ -448,12 +560,14 @@ def test_materialised_consumer_mismatch_is_not_hidden(self) -> None: class DeployActionOrderingTests(unittest.TestCase): """Keep credential refreshes on both sides of KSail-owned mutations.""" - def test_refresh_precedes_reconcile_and_is_reasserted_after_update(self) -> None: + def test_consumer_staging_precedes_publish_and_is_reasserted_after_update( + self, + ) -> None: action = ACTION.read_text(encoding="utf-8") - first_refresh = action.index("id: preflight_flux_ghcr_auth") + first_refresh = action.index("id: stage_flux_ghcr_auth") push = action.index("run: ksail --config ksail.prod.yaml workload push") - post_push_refresh = action.index("id: reassert_flux_ghcr_auth_after_push") + post_push_refresh = action.index("id: verify_flux_ghcr_auth_after_push") reconcile = action.index("id: reconcile") cluster_update = action.index( "run: ksail --config ksail.prod.yaml cluster update" @@ -465,14 +579,22 @@ def test_refresh_precedes_reconcile_and_is_reasserted_after_update(self) -> None self.assertLess(post_push_refresh, reconcile) self.assertLess(reconcile, cluster_update) self.assertLess(cluster_update, final_refresh) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh\n", + action[first_refresh:push], + ) + self.assertNotIn( + "--check-only", + action[first_refresh:push], + ) self.assertIn( "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only", - action, + action[post_push_refresh:reconcile], ) final_refresh_step = action[final_refresh:] self.assertIn("always() &&", final_refresh_step) self.assertIn( - "steps.reassert_flux_ghcr_auth_after_push.outcome == 'success'", + "steps.verify_flux_ghcr_auth_after_push.outcome == 'success'", final_refresh_step, ) self.assertIn( @@ -481,7 +603,7 @@ def test_refresh_precedes_reconcile_and_is_reasserted_after_update(self) -> None ) self.assertEqual(action.count("scripts/refresh-flux-ghcr-auth.sh"), 3) - def test_disaster_rebuild_preflights_before_mutation_and_refreshes_before_reconcile( + def test_disaster_rebuild_preflights_then_stages_before_publish( self, ) -> None: workflow = DR_REBUILD.read_text(encoding="utf-8") @@ -492,19 +614,50 @@ def test_disaster_rebuild_preflights_before_mutation_and_refreshes_before_reconc cluster_create = workflow.index( "run: ksail --config ksail.prod.yaml cluster create" ) + stage = workflow.index("id: stage_flux_ghcr_auth") push = workflow.index("run: ksail --config ksail.prod.yaml workload push") - refresh = workflow.index( - "run: ./scripts/refresh-flux-ghcr-auth.sh\n" - ) + verify = workflow.index("id: verify_flux_ghcr_auth_after_push") + fanout_verify = workflow.index("id: verify_flux_ghcr_fanout") reconcile = workflow.index( "run: ksail --config ksail.prod.yaml workload reconcile" ) self.assertLess(preflight, cluster_create) - self.assertLess(cluster_create, push) - self.assertLess(push, refresh) - self.assertLess(refresh, reconcile) - self.assertEqual(workflow.count("scripts/refresh-flux-ghcr-auth.sh"), 2) + self.assertLess(cluster_create, stage) + self.assertLess(stage, push) + self.assertLess(push, verify) + self.assertLess(verify, reconcile) + self.assertLess(reconcile, fanout_verify) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout", + workflow[stage:push], + ) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only", + workflow[verify:reconcile], + ) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh\n", + workflow[fanout_verify:], + ) + self.assertEqual(workflow.count("scripts/refresh-flux-ghcr-auth.sh"), 4) + + +class RequiredPackageCoverageTests(unittest.TestCase): + """Keep pinned private provider references in the live GHCR preflight.""" + + def test_provider_upjet_unifi_reference_is_preflighted(self) -> None: + manifest = PROVIDER_UPJET_UNIFI.read_text(encoding="utf-8") + helper = HELPER.read_text(encoding="utf-8") + + package_line = next( + line.strip() + for line in manifest.splitlines() + if line.strip().startswith("package: ghcr.io/") + ) + package_reference = package_line.removeprefix("package: ghcr.io/") + + self.assertIn(f'"{package_reference}"', helper) if __name__ == "__main__": From efdd4dabf70ef224c04a8c940e185406fb7a3533 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 01:45:45 +0200 Subject: [PATCH 08/16] fix(dr): reassert GHCR auth after vault restore --- .github/workflows/dr-rebuild.yaml | 8 ++++++++ AGENTS.md | 2 +- docs/dr/runbook.md | 3 ++- scripts/tests/test_refresh_flux_ghcr_auth.py | 14 +++++++++++++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index 5a7918154..bcdb58e7b 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -398,6 +398,14 @@ jobs: echo "::endgroup::" echo "✅ OpenBao restored from the raft snapshot mirror." + - name: 🔑 Reassert GHCR credential after OpenBao restore + id: reassert_flux_ghcr_after_restore + if: ${{ inputs.restore }} + # The raft snapshot may contain an older infrastructure/ghcr/auth value. + # Re-stage Git/SOPS in PushSecret-first order, wait for every consumer, + # compare the materialized payloads, and only then reassert root auth. + run: ./scripts/refresh-flux-ghcr-auth.sh + - name: 🔑 Refresh CI deploy credentials (KUBE_CONFIG / TALOS_CONFIG) env: DR_GH_ADMIN_TOKEN: ${{ secrets.DR_GH_ADMIN_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index c779fcd35..3e040e724 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -148,7 +148,7 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 6. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. 7. `scripts/refresh-flux-ghcr-auth.sh --check-only` revalidates the newly-published artifact without mutating the cluster. 8. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. -9. After `cluster update`, the full bridge reasserts the Git/SOPS credential in case KSail rewrote its managed root Secret. +9. After `cluster update`, the full bridge reasserts the Git/SOPS credential in case KSail rewrote its managed root Secret. DR also runs it after an OpenBao raft restore because the snapshot may contain an older GHCR value. **Key differences from local:** diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index 7ab242a16..8800ddd19 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -520,7 +520,8 @@ The production deploy closes the bootstrap loop in this order: 3. Push and sign the artifact with `GHCR_TOKEN`, revalidate the newly-published artifact with `--check-only`, and only then explicitly reconcile Flux. DR runs the full bridge again after every Flux Kustomization is Ready, proving - that bootstrap mode completed the entire fan-out. + that bootstrap mode completed the entire fan-out, and once more after an + OpenBao raft restore so a snapshot cannot rematerialise an older credential. 4. Re-run the bridge after `cluster update`, because KSail can rewrite its managed root Secret when another cluster setting changes; the bridge also re-verifies the downstream fan-out. diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index c4293311b..1caf11868 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -618,6 +618,12 @@ def test_disaster_rebuild_preflights_then_stages_before_publish( push = workflow.index("run: ksail --config ksail.prod.yaml workload push") verify = workflow.index("id: verify_flux_ghcr_auth_after_push") fanout_verify = workflow.index("id: verify_flux_ghcr_fanout") + openbao_restore = workflow.index( + "name: 🔐 Restore OpenBao from the R2 snapshot mirror" + ) + post_restore_verify = workflow.index( + "id: reassert_flux_ghcr_after_restore" + ) reconcile = workflow.index( "run: ksail --config ksail.prod.yaml workload reconcile" ) @@ -628,6 +634,8 @@ def test_disaster_rebuild_preflights_then_stages_before_publish( self.assertLess(push, verify) self.assertLess(verify, reconcile) self.assertLess(reconcile, fanout_verify) + self.assertLess(fanout_verify, openbao_restore) + self.assertLess(openbao_restore, post_restore_verify) self.assertIn( "run: ./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout", workflow[stage:push], @@ -640,7 +648,11 @@ def test_disaster_rebuild_preflights_then_stages_before_publish( "run: ./scripts/refresh-flux-ghcr-auth.sh\n", workflow[fanout_verify:], ) - self.assertEqual(workflow.count("scripts/refresh-flux-ghcr-auth.sh"), 4) + self.assertIn( + "if: ${{ inputs.restore }}", + workflow[post_restore_verify:], + ) + self.assertEqual(workflow.count("scripts/refresh-flux-ghcr-auth.sh"), 5) class RequiredPackageCoverageTests(unittest.TestCase): From 7eccf203c50c6d351b8cb4108a4ab48722f12fa3 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 01:53:34 +0200 Subject: [PATCH 09/16] docs(deploy): document GHCR bridge coverage --- scripts/refresh-flux-ghcr-auth.sh | 3 +++ scripts/tests/test_refresh_flux_ghcr_auth.py | 23 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index d83750b6e..b6069c6c2 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -63,6 +63,7 @@ variables_patch_file="${work_dir}/variables-patch.json" expected_normalized="${work_dir}/expected-normalized.json" fanout_api_resources="${work_dir}/fanout-api-resources.txt" +# Force an ESO resource to reconcile and observe a post-annotation Ready edge. force_sync_resource() { local kind="$1" local namespace="$2" @@ -121,6 +122,7 @@ force_sync_resource() { return 1 } +# Verify that a namespace's materialized GHCR Secret matches the SOPS source. verify_consumer_secret() { local namespace="$1" local secret_file="${work_dir}/consumer-${namespace}.json" @@ -264,6 +266,7 @@ base64 < "${docker_config}" \ | jq -Rs '{data: {".dockerconfigjson": .}}' \ > "${patch_file}" +# Patch only the root Flux Secret payload, preserving KSail ownership metadata. patch_root_secret() { kubectl \ --context "${KUBE_CONTEXT}" \ diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index 1caf11868..6b24f1c6e 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -31,6 +31,7 @@ class RefreshFluxGhcrAuthTests(unittest.TestCase): """Exercise the helper with fake external commands and no real secrets.""" def setUp(self) -> None: + """Create isolated command fakes and capture files for each test.""" self.temp_dir = tempfile.TemporaryDirectory() self.addCleanup(self.temp_dir.cleanup) self.workspace = Path(self.temp_dir.name) @@ -270,6 +271,7 @@ def setUp(self) -> None: ) def _write_executable(self, name: str, body: str) -> None: + """Install an executable command fake in the isolated test PATH.""" path = self.bin_dir / name path.write_text(textwrap.dedent(body).lstrip(), encoding="utf-8") path.chmod(0o755) @@ -280,6 +282,7 @@ def _run_helper( helper_args: tuple[str, ...] = (), **environment_overrides: str, ) -> subprocess.CompletedProcess[str]: + """Run the credential bridge against a supplied Docker config fixture.""" self.decrypted_config.write_text(json.dumps(config), encoding="utf-8") for marker in ( self.patch_capture, @@ -320,6 +323,7 @@ def _run_helper( @staticmethod def _valid_config() -> dict[str, object]: + """Return a valid explicit GHCR Docker authentication fixture.""" return { "auths": { "ghcr.io": { @@ -330,6 +334,7 @@ def _valid_config() -> dict[str, object]: } def test_refreshes_root_and_fanout_without_leaking_plaintext(self) -> None: + """Refresh every credential consumer without exposing the pull token.""" config = self._valid_config() result = self._run_helper(config) @@ -370,6 +375,7 @@ def test_refreshes_root_and_fanout_without_leaking_plaintext(self) -> None: ) def test_accepts_standard_auth_only_docker_config(self) -> None: + """Accept Docker configs that store only the standard encoded auth field.""" auth = base64.b64encode(b"devantler:fixture-secret-token").decode() config = {"auths": {"ghcr.io": {"auth": auth}}} @@ -381,6 +387,7 @@ def test_accepts_standard_auth_only_docker_config(self) -> None: self.assertEqual(json.loads(base64.b64decode(encoded)), config) def test_accepts_matching_explicit_and_encoded_auth(self) -> None: + """Accept matching explicit and encoded GHCR credentials.""" config = self._valid_config() registry_auth = config["auths"]["ghcr.io"] registry_auth["auth"] = base64.b64encode( @@ -392,6 +399,7 @@ def test_accepts_matching_explicit_and_encoded_auth(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) def test_check_only_preflights_without_patching(self) -> None: + """Keep registry preflight mode free of Kubernetes mutations.""" result = self._run_helper(self._valid_config(), ("--check-only",)) self.assertEqual(result.returncode, 0, result.stderr) @@ -399,6 +407,7 @@ def test_check_only_preflights_without_patching(self) -> None: self.assertFalse(self.patch_capture.exists()) def test_missing_or_malformed_registry_auth_fails_closed(self) -> None: + """Reject missing, malformed, empty, or contradictory GHCR credentials.""" invalid_configs: list[object] = [ {"auths": {"ghcr.io": {"username": "devantler"}}}, {"auths": {"ghcr.io": {"username": "", "password": "token"}}}, @@ -425,6 +434,7 @@ def test_missing_or_malformed_registry_auth_fails_closed(self) -> None: self.assertFalse(self.kubectl_called.exists()) def test_registry_denial_prevents_cluster_patch(self) -> None: + """Leave cluster credentials untouched when GHCR denies a manifest read.""" result = self._run_helper( self._valid_config(), FAKE_CURL_DENY_REPOSITORY="devantler-tech/platform/manifests", @@ -436,6 +446,7 @@ def test_registry_denial_prevents_cluster_patch(self) -> None: def test_token_success_without_registry_read_access_prevents_cluster_patch( self, ) -> None: + """Require package read access even when the token exchange succeeds.""" result = self._run_helper( self._valid_config(), FAKE_CURL_DENY_REPOSITORY="devantler-tech/wedding-app", @@ -445,12 +456,14 @@ def test_token_success_without_registry_read_access_prevents_cluster_patch( self.assertFalse(self.kubectl_called.exists()) def test_cluster_patch_failure_is_not_hidden(self) -> None: + """Propagate a failure to patch the root Flux pull Secret.""" result = self._run_helper(self._valid_config(), FAKE_KUBECTL_FAIL="true") self.assertEqual(result.returncode, 43) self.assertTrue(self.kubectl_called.exists()) def test_fresh_cluster_without_variables_base_skips_existing_fanout(self) -> None: + """Bootstrap only root auth before a fresh cluster creates its fan-out.""" result = self._run_helper( self._valid_config(), ("--allow-incomplete-fanout",), @@ -463,6 +476,7 @@ def test_fresh_cluster_without_variables_base_skips_existing_fanout(self) -> Non self.assertFalse(self.fanout_log.exists()) def test_missing_variables_base_fails_closed_without_bootstrap_mode(self) -> None: + """Require the explicit DR flag when the credential fan-out is absent.""" result = self._run_helper( self._valid_config(), FAKE_VARIABLES_BASE_ABSENT="true", @@ -474,6 +488,7 @@ def test_missing_variables_base_fails_closed_without_bootstrap_mode(self) -> Non def test_partial_bootstrap_repairs_root_without_forcing_missing_fanout( self, ) -> None: + """Stage DR root auth without force-syncing an incomplete fan-out.""" missing_resources = [ "pushsecret/flux-system/seed-ghcr", "externalsecret/wedding-app/ghcr-auth", @@ -495,6 +510,7 @@ def test_partial_bootstrap_repairs_root_without_forcing_missing_fanout( self.assertIn("first reconcile will complete", result.stdout) def test_partial_fanout_fails_closed_without_bootstrap_mode(self) -> None: + """Reject a partial fan-out during a normal production deployment.""" result = self._run_helper( self._valid_config(), FAKE_MISSING_FANOUT_RESOURCE="externalsecret/kyverno/ghcr-auth", @@ -505,6 +521,7 @@ def test_partial_fanout_fails_closed_without_bootstrap_mode(self) -> None: self.assertFalse(self.fanout_log.exists()) def test_partial_bootstrap_without_eso_crds_repairs_root(self) -> None: + """Permit DR root repair before External Secrets CRDs exist.""" result = self._run_helper( self._valid_config(), ("--allow-incomplete-fanout",), @@ -517,6 +534,7 @@ def test_partial_bootstrap_without_eso_crds_repairs_root(self) -> None: self.assertFalse(self.fanout_log.exists()) def test_pushsecret_sync_failure_is_not_hidden(self) -> None: + """Keep root auth unchanged when the OpenBao seed does not reconcile.""" result = self._run_helper( self._valid_config(), FAKE_SYNC_STALL_RESOURCE="pushsecret/flux-system/seed-ghcr", @@ -531,6 +549,7 @@ def test_pushsecret_sync_failure_is_not_hidden(self) -> None: self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) def test_same_second_sync_accepts_controller_resource_version_edge(self) -> None: + """Accept a controller update when refreshTime has one-second precision.""" result = self._run_helper( self._valid_config(), FAKE_SYNC_SAME_REFRESH_TIME="true", @@ -540,6 +559,7 @@ def test_same_second_sync_accepts_controller_resource_version_edge(self) -> None self.assertTrue(self.patch_capture.exists()) def test_materialised_consumer_mismatch_is_not_hidden(self) -> None: + """Reject fan-out completion when a workload Secret has stale content.""" result = self._run_helper( self._valid_config(), FAKE_CONSUMER_MISMATCH_NAMESPACE="wedding-app", @@ -563,6 +583,7 @@ class DeployActionOrderingTests(unittest.TestCase): def test_consumer_staging_precedes_publish_and_is_reasserted_after_update( self, ) -> None: + """Keep full credential staging before publish and after cluster update.""" action = ACTION.read_text(encoding="utf-8") first_refresh = action.index("id: stage_flux_ghcr_auth") @@ -606,6 +627,7 @@ def test_consumer_staging_precedes_publish_and_is_reasserted_after_update( def test_disaster_rebuild_preflights_then_stages_before_publish( self, ) -> None: + """Keep DR credential checks around publish and OpenBao restoration.""" workflow = DR_REBUILD.read_text(encoding="utf-8") preflight = workflow.index( @@ -659,6 +681,7 @@ class RequiredPackageCoverageTests(unittest.TestCase): """Keep pinned private provider references in the live GHCR preflight.""" def test_provider_upjet_unifi_reference_is_preflighted(self) -> None: + """Require the live private provider package in the GHCR preflight.""" manifest = PROVIDER_UPJET_UNIFI.read_text(encoding="utf-8") helper = HELPER.read_text(encoding="utf-8") From 677a958b8e00e365e74c91ea73530c7661482bd0 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 01:56:35 +0200 Subject: [PATCH 10/16] docs(deploy): remove orphaned recovery text --- AGENTS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3e040e724..4bbe92cb9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -184,9 +184,9 @@ force-syncs `seed-ghcr` into OpenBao, force-syncs the tenant/Kyverno ExternalSecrets, and verifies their materialised `ghcr-auth` payloads before switching root Flux auth. Only explicit DR bootstrap mode may repair root auth after staging `variables-base` while the fan-out is incomplete; DR must run the -full verifier after Flux converges. -apps reconcile. A direct credential commit to `main` still needs a -manual `CD` workflow dispatch because direct pushes bypass the merge-queue deploy. +full verifier after Flux converges. A direct credential commit to `main` still +needs a manual `CD` workflow dispatch because direct pushes bypass the merge-queue +deploy. Talos node registry auth still derives from `GHCR_TOKEN`; consolidating that remaining pull path stays tracked by #2613 and the KSail credential work. From 520024ca39cb3529b778881f62ee123a6f41e4ee Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 02:44:50 +0200 Subject: [PATCH 11/16] fix(deploy): synchronize Talos GHCR pull auth --- .github/actions/deploy-prod/action.yml | 35 +- .github/workflows/ci.yaml | 19 +- .github/workflows/dr-rebuild.yaml | 24 +- AGENTS.md | 33 +- docs/dr/crypto-custody.md | 3 +- docs/dr/runbook.md | 65 +- ksail.prod.yaml | 2 +- scripts/ghcr-auth-lib.sh | 73 ++ scripts/refresh-flux-ghcr-auth.sh | 225 ++++-- scripts/run-ksail-prod-with-pull-auth.sh | 59 ++ scripts/tests/test_refresh_flux_ghcr_auth.py | 738 ++++++++++++++++++- talos/cluster/authenticate-ghcr-pulls.yaml | 52 +- talos/cluster/mark-ghcr-pull-revision.yaml | 14 + 13 files changed, 1181 insertions(+), 161 deletions(-) create mode 100644 scripts/ghcr-auth-lib.sh create mode 100755 scripts/run-ksail-prod-with-pull-auth.sh create mode 100644 talos/cluster/mark-ghcr-pull-revision.yaml diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 2b021e944..14e79f9b6 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -19,7 +19,7 @@ inputs: description: talosconfig for the prod cluster (secrets.TALOS_CONFIG) used by `ksail cluster update`. required: true ghcr-token: - description: GHCR PAT with write:packages (secrets.GHCR_TOKEN) for OCI push, signing, and Talos/KSail compatibility; Flux pulls use the Git/SOPS credential. + description: GHCR PAT with write:packages (secrets.GHCR_TOKEN) for OCI push and signing; every pull path uses the Git/SOPS credential. required: true hcloud-token: description: Hetzner Cloud API token (secrets.HCLOUD_TOKEN) used by the KSail Hetzner provider and CCM/CSI. @@ -50,6 +50,19 @@ runs: sudo install /tmp/ksail /usr/local/bin/ksail ksail --version + - name: ⚙️ Setup talosctl + # refresh-flux-ghcr-auth.sh uses the host API for a secret-safe, + # per-node RegistryAuthConfig patch and exact image-pull proof. + shell: bash + env: + # Keep in step with spec.cluster.talos.version in ksail.prod.yaml. + # renovate: datasource=github-releases depName=siderolabs/talos extractVersion=^v(?.+)$ + TALOS_VERSION: "1.13.5" + run: | + curl -fsSL "https://github.com/siderolabs/talos/releases/download/v${TALOS_VERSION}/talosctl-linux-amd64" -o /tmp/talosctl + sudo install /tmp/talosctl /usr/local/bin/talosctl + talosctl version --client + - name: 🔐 Create SOPS Age key shell: bash env: @@ -128,10 +141,10 @@ runs: - name: 🔑 Stage Flux and consumer GHCR pull credential id: stage_flux_ghcr_auth - # Validate all seven packages, then synchronously stage and verify the whole - # pull-credential fan-out BEFORE publishing latest. If the mutable tag is - # observed by Flux immediately after push, every consumer already has the - # same known-readable credential. + # Validate all seven packages, synchronize and prove the exact live image + # pull on every stale Talos node, then stage and verify the Kubernetes fan-out + # BEFORE publishing latest. Every pull consumer has one known-readable + # credential before the mutable tag can be observed. shell: bash run: ./scripts/refresh-flux-ghcr-auth.sh @@ -148,7 +161,7 @@ runs: GITHUB_ACTOR: ${{ github.actor }} GHCR_TOKEN: ${{ inputs.ghcr-token }} HCLOUD_TOKEN: ${{ inputs.hcloud-token }} - run: ksail --config ksail.prod.yaml workload push + run: ./scripts/run-ksail-prod-with-pull-auth.sh workload push - name: ⚙️ Install cosign # sigstore/cosign-installer is the official installer action — it @@ -305,9 +318,8 @@ runs: shell: bash env: GITHUB_ACTOR: ${{ github.actor }} - GHCR_TOKEN: ${{ inputs.ghcr-token }} HCLOUD_TOKEN: ${{ inputs.hcloud-token }} - run: ksail --config ksail.prod.yaml workload reconcile + run: ./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile - name: 🩺 Diagnose Flux on failure if: failure() && steps.reconcile.outcome == 'failure' @@ -330,18 +342,17 @@ runs: # release). shell: bash env: - GHCR_TOKEN: ${{ inputs.ghcr-token }} HCLOUD_TOKEN: ${{ inputs.hcloud-token }} # ksail env-expands ${WG_SERVER_PRIVATE_KEY} in the talos patches at # load time; an unset var renders an empty key and Talos rejects the # whole machine config ('private key is invalid: wrong key "" length: 0'). WG_SERVER_PRIVATE_KEY: ${{ inputs.wg-server-private-key }} - run: ksail --config ksail.prod.yaml cluster update + run: ./scripts/run-ksail-prod-with-pull-auth.sh cluster update - name: 🔑 Reassert Flux and consumer GHCR pull credential id: reassert_flux_ghcr_auth - # cluster update can rewrite KSail's root Secret from GHCR_TOKEN when an - # unrelated configuration change triggers setup. Reassert Git/SOPS after + # cluster update is already wrapped with Git/SOPS auth, but reassert the + # full transaction afterward in case KSail fails part-way. Reassert after # its synchronous consumer fan-out when that update fails part-way, but # only after the publish, signing, attestation, post-publish auth check, # and Flux reconcile all succeeded. Never mutate auth as cleanup for a diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 415f3df74..f573f4ea0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,6 +42,8 @@ jobs: - 'scripts/validate-naming.py' - 'scripts/validate-embedded-json.py' - 'scripts/refresh-flux-ghcr-auth.sh' + - 'scripts/ghcr-auth-lib.sh' + - 'scripts/run-ksail-prod-with-pull-auth.sh' - 'scripts/tests/test_refresh_flux_ghcr_auth.py' - '.github/actions/deploy-prod/**' - '.github/workflows/dr-rebuild.yaml' @@ -73,7 +75,14 @@ jobs: # Pure local tests with fake ksail/docker/kubectl commands: no SOPS key, # registry credential, cluster access, or network is used. run: | - bash -n scripts/refresh-flux-ghcr-auth.sh + bash -n \ + scripts/ghcr-auth-lib.sh \ + scripts/refresh-flux-ghcr-auth.sh \ + scripts/run-ksail-prod-with-pull-auth.sh + shellcheck \ + scripts/ghcr-auth-lib.sh \ + scripts/refresh-flux-ghcr-auth.sh \ + scripts/run-ksail-prod-with-pull-auth.sh python3 -m unittest scripts.tests.test_refresh_flux_ghcr_auth - name: 🧩 Validate embedded JSON blobs @@ -241,12 +250,16 @@ jobs: # Placeholder key material for env-expanded patch values (ksail # expands these from environment secrets at deploy time — see - # talos/control-planes/wireguard.yaml and talos/cluster/registry-auth.yaml). + # talos/control-planes/wireguard.yaml and + # talos/cluster/authenticate-ghcr-pulls.yaml and + # talos/cluster/mark-ghcr-pull-revision.yaml). # Real-format but throwaway, so validation exercises the expansion # path without any real secret on the PR event. WG_SERVER_PRIVATE_KEY="$(openssl rand -base64 32)" + GHCR_USERNAME="placeholder-ghcr-user" GHCR_TOKEN="placeholder-ghcr-token" - export WG_SERVER_PRIVATE_KEY GHCR_TOKEN + GHCR_PULL_REVISION="$(printf '%s\n' 'ENC[ci-placeholder-ciphertext]' | shasum -a 256 | awk '{print $1}')" + export WG_SERVER_PRIVATE_KEY GHCR_USERNAME GHCR_TOKEN GHCR_PULL_REVISION validate_overlay() { local dir="$1" mode="$2" diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index bcdb58e7b..db34855f9 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -85,6 +85,17 @@ jobs: sudo install /tmp/ksail /usr/local/bin/ksail ksail --version + - name: ⚙️ Setup talosctl + # The GHCR bridge verifies RegistryAuthConfig on each stale node. + env: + # Keep in step with spec.cluster.talos.version in ksail.prod.yaml. + # renovate: datasource=github-releases depName=siderolabs/talos extractVersion=^v(?.+)$ + TALOS_VERSION: "1.13.5" + run: | + curl -fsSL "https://github.com/siderolabs/talos/releases/download/v${TALOS_VERSION}/talosctl-linux-amd64" -o /tmp/talosctl + sudo install /tmp/talosctl /usr/local/bin/talosctl + talosctl version --client + - name: 🔐 Create SOPS Age key env: SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }} @@ -105,9 +116,8 @@ jobs: # talosconfig (~/.talos/config) on this runner — every later step uses # those, so this workflow does NOT depend on the (now stale) # KUBE_CONFIG / TALOS_CONFIG environment secrets. - run: ksail --config ksail.prod.yaml cluster create + run: ./scripts/run-ksail-prod-with-pull-auth.sh cluster create env: - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} # ksail env-expands ${WG_SERVER_PRIVATE_KEY} in the talos patches at # load time (talos/control-planes/wireguard.yaml); an unset var @@ -116,14 +126,15 @@ jobs: - name: 🔑 Stage Flux and consumer GHCR pull credential id: stage_flux_ghcr_auth - # cluster create bootstraps with GHCR_TOKEN for KSail compatibility. - # Make Git/SOPS authoritative before the mutable tag is published. A + # cluster create already uses the Git/SOPS pull token. Reassert it on + # every current Talos node and make Git/SOPS authoritative before the + # mutable tag is published. A # partial bootstrap may have variables-base but no ESO resources yet; # the bridge safely repairs root auth so Flux can create them. run: ./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout - name: 📦 Push manifests to GHCR - run: ksail --config ksail.prod.yaml workload push + run: ./scripts/run-ksail-prod-with-pull-auth.sh workload push env: GITHUB_ACTOR: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} @@ -136,10 +147,9 @@ jobs: run: ./scripts/refresh-flux-ghcr-auth.sh --check-only - name: 🔁 Trigger Flux reconciliation - run: ksail --config ksail.prod.yaml workload reconcile + run: ./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile env: GITHUB_ACTOR: ${{ github.actor }} - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - name: ⏳ Wait for Flux to settle diff --git a/AGENTS.md b/AGENTS.md index 4bbe92cb9..fd07906f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -143,12 +143,12 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 1. Merging a PR through the merge queue runs the `deploy-prod` job in `ci.yaml` (the normal path). A direct push to `main` bypasses the queue, so deploy it manually by running the `CD` workflow (`cd.yaml`, `workflow_dispatch`). Both run the same `ksail` steps below. 2. The `deploy-prod` composite action (shared by both paths) uses `ksail --config ksail.prod.yaml` to target the committed prod config. 3. `ksail.prod.yaml` has `kustomizationFile: clusters/prod`, so KSail/Flux use `k8s/clusters/prod/kustomization.yaml` as the entry point — no root `k8s/kustomization.yaml` or file rewriting is needed. -4. `ksail --config ksail.prod.yaml cluster create` (first run) or `cluster update` (subsequent runs) provisions / reconciles the Hetzner servers, Talos, CCM, and CSI. -5. The bridge decrypts only the Git/SOPS pull credential, performs real OCI manifest reads for all seven private consumers (the Platform and tenant manifest artifacts, both tenant application images, and the KSail plus provider-upjet-unifi packages used by Kyverno verification), updates `variables-base`, force-syncs and verifies the PushSecret plus tenant/Kyverno ExternalSecrets, and only then reasserts root auth — all before a mutable `latest` tag is published. The DR workflow first runs `--check-only` before creating infrastructure, then uses explicit `--allow-incomplete-fanout` bootstrap mode after cluster creation and requires a full bridge pass after Flux converges. -6. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. +4. `scripts/run-ksail-prod-with-pull-auth.sh cluster create|update` provisions / reconciles the Hetzner servers, Talos, CCM, and CSI with the Git/SOPS pull credential; the wrapper also passes a SOPS-ciphertext revision so token-only rotations refresh the Cluster Autoscaler machine template. +5. The bridge decrypts only the Git/SOPS pull credential and performs real OCI manifest reads for all seven private consumers (the Platform and tenant manifest artifacts, both tenant application images, and the KSail plus provider-upjet-unifi packages used by Kyverno verification). On nodes whose verified revision is stale, it applies Talos `RegistryAuthConfig` workers-first, proves the exact private KSail image pull, and only then records the verified revision. It then updates `variables-base`, force-syncs and verifies the PushSecret plus tenant/Kyverno ExternalSecrets, and finally reasserts root auth — all before a mutable `latest` tag is published. The DR workflow first runs `--check-only` before creating infrastructure, then uses explicit `--allow-incomplete-fanout` bootstrap mode after cluster creation and requires a full bridge pass after Flux converges. +6. `scripts/run-ksail-prod-with-pull-auth.sh workload push` packages manifests and pushes them with the separate Actions write token. 7. `scripts/refresh-flux-ghcr-auth.sh --check-only` revalidates the newly-published artifact without mutating the cluster. -8. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. -9. After `cluster update`, the full bridge reasserts the Git/SOPS credential in case KSail rewrote its managed root Secret. DR also runs it after an OpenBao raft restore because the snapshot may contain an older GHCR value. +8. `scripts/run-ksail-prod-with-pull-auth.sh workload reconcile` triggers Flux with Git/SOPS pull auth. +9. After `cluster update`, the full bridge reasserts every pull path in case a partial update or older managed state was applied. DR also runs it after an OpenBao raft restore because the snapshot may contain an older GHCR value. **Key differences from local:** @@ -160,7 +160,7 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o ### Dual-Provider Model - **Local / CI:** `ksail cluster create` → Talos + Docker provider → local OCI registry → `ksail workload push` / `reconcile`. -- **Production:** `ksail --config ksail.prod.yaml cluster create|update` → Talos + Hetzner provider → Hetzner CCM + CSI installed by KSail → `ksail --config ksail.prod.yaml workload push` to GHCR → `workload reconcile`. +- **Production:** `scripts/run-ksail-prod-with-pull-auth.sh cluster create|update` → Talos + Hetzner provider → Hetzner CCM + CSI installed by KSail → the same wrapper's `workload push` to GHCR → `workload reconcile`. ## CI/CD Pipelines @@ -170,11 +170,12 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o **Required GitHub Secrets:** -- `GHCR_TOKEN` — long-lived PAT (owner: `devantler`) with `write:packages` scope, used for OCI push/signing and retained for KSail/Talos compatibility. It is **not** the authoritative Flux pull credential. +- `GHCR_TOKEN` — long-lived PAT (owner: `devantler`) with `write:packages` scope, used only for OCI push/signing. It is **not** a pull credential. - `SOPS_AGE_KEY` — Age private key for SOPS secret decryption. - `HCLOUD_TOKEN` — Hetzner Cloud API token (read/write), used by the KSail Hetzner provider and by the Hetzner CCM / CSI at runtime. -The authoritative **Flux and tenant** GHCR pull credential is +The authoritative **production pull** credential for Flux, tenants, Kyverno, +and Talos hosts is `stringData.ghcr_dockerconfigjson` in `k8s/bases/bootstrap/secret.enc.yaml`. The deploy bridge refreshes `flux-system/ksail-registry-credentials` from that value before Flux must fetch @@ -187,8 +188,10 @@ after staging `variables-base` while the fan-out is incomplete; DR must run the full verifier after Flux converges. A direct credential commit to `main` still needs a manual `CD` workflow dispatch because direct pushes bypass the merge-queue deploy. -Talos node registry auth still derives from `GHCR_TOKEN`; consolidating that -remaining pull path stays tracked by #2613 and the KSail credential work. +The lifecycle wrapper injects the same username/token into KSail's local +registry and Talos patches. A non-secret hash of the committed SOPS ciphertext +is the desired machine-template revision; the bridge stores a separate verified +revision on each existing node only after an exact image pull succeeds. **Required GitHub Variables:** none. @@ -323,10 +326,14 @@ With the KSail Hetzner provider the cluster is cattle — rebuild it in place: ```bash export HCLOUD_TOKEN=... -ksail --config ksail.prod.yaml cluster update # scales / re-provisions missing nodes +export WG_SERVER_PRIVATE_KEY=... +export SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt +export GHCR_TOKEN=... # publication only +export GITHUB_ACTOR=devantler +./scripts/run-ksail-prod-with-pull-auth.sh cluster update # For a full rebuild from zero, see docs/dr/runbook.md scenario 4. -ksail --config ksail.prod.yaml workload push -ksail --config ksail.prod.yaml workload reconcile +./scripts/run-ksail-prod-with-pull-auth.sh workload push +./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile ``` ### Tool Reinstallation diff --git a/docs/dr/crypto-custody.md b/docs/dr/crypto-custody.md index 4e9972dc5..4cd961664 100644 --- a/docs/dr/crypto-custody.md +++ b/docs/dr/crypto-custody.md @@ -120,7 +120,8 @@ posture is: 1. **Do not commit it to git.** It is `.gitignore`d via the default `~/.talos/config` location. 2. **Keep one copy on each operator workstation** that needs to manage - nodes. Replacing a workstation = re-run `ksail --config ksail.prod.yaml cluster update` + nodes. Replacing a workstation = re-run + `./scripts/run-ksail-prod-with-pull-auth.sh cluster update` (which regenerates the local `~/.talos/config` from the cluster's running PKI), or use `talosctl config merge ` against a backup file. Talos does not have SSH-equivalent — see DR Scenario 4 diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index 8800ddd19..9a24b46fe 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -48,6 +48,14 @@ these simultaneously and you cannot recover. > and updating the unifi tenant's `cluster_wg_peer_public_key` to the new > public half. +> **Manual production lifecycle prerequisite:** use +> `scripts/run-ksail-prod-with-pull-auth.sh`, never a bare +> `ksail --config ksail.prod.yaml` create/update/push/reconcile command. The +> wrapper injects the authoritative SOPS pull credential and its rotation +> marker. Export `HCLOUD_TOKEN`, `WG_SERVER_PRIVATE_KEY`, and a usable SOPS Age +> key for create/update; publication additionally needs `GHCR_TOKEN` and +> `GITHUB_ACTOR`. + --- ## Scenario 1 — Single node loss @@ -62,7 +70,7 @@ kubectl get pods -A --field-selector=status.phase!=Running kubectl get pdb -A # all should show ALLOWED-DISRUPTIONS=1 # Replace the failed node (re-runs Hetzner provisioning for missing members) -ksail --config ksail.prod.yaml cluster update +./scripts/run-ksail-prod-with-pull-auth.sh cluster update ``` If any workload is stuck in Pending because all replicas were on the dead @@ -101,7 +109,7 @@ kubectl get pdb -A kubectl get deploy -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}\t{.spec.strategy.rollingUpdate.maxUnavailable}{"\n"}{end}' # Apply the upgrade (in-place rolling Talos OS + Kubernetes upgrade) -ksail --config ksail.prod.yaml cluster update +./scripts/run-ksail-prod-with-pull-auth.sh cluster update ``` If anything reports `maxUnavailable` other than `0`, that workload was @@ -156,26 +164,31 @@ Flux reconciliation. ```bash # 1. Set credentials locally export HCLOUD_TOKEN= +export WG_SERVER_PRIVATE_KEY= export GHCR_TOKEN= +export GITHUB_ACTOR=devantler export SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt # points at the env's Age key +talosctl version --client # must be installed; use the prod-pinned v1.13.5 # 2. Prove the Git/SOPS pull credential can read every private package before # creating infrastructure or publishing a mutable latest tag ./scripts/refresh-flux-ghcr-auth.sh --check-only # 3. Boot a fresh cluster (ksail handles Talos boot, CCM, CSI, kubeconfig) -ksail --config ksail.prod.yaml cluster create +./scripts/run-ksail-prod-with-pull-auth.sh cluster create # 4. Bootstrap Flux from this repo -ksail --config ksail.prod.yaml workload push # packages -> GHCR -./scripts/refresh-flux-ghcr-auth.sh # Git/SOPS -> root + existing fan-out -ksail --config ksail.prod.yaml workload reconcile # Flux pulls and applies +./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout +./scripts/run-ksail-prod-with-pull-auth.sh workload push +./scripts/refresh-flux-ghcr-auth.sh --check-only +./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile # 5. Wait for Flux to settle flux get kustomizations -A # Re-run if any are NotReady; expect convergence in 10-15 minutes +./scripts/refresh-flux-ghcr-auth.sh # prove completed fan-out + every stale node -# 4b. ONLY if the OpenBao raft-snapshot recovery was impossible (no snapshot +# 6. ONLY if the OpenBao raft-snapshot recovery was impossible (no snapshot # in R2 — the vault came up fresh): re-feed the user-fed secrets that # SOPS deliberately does not seed (see the push-secret-seed-* files in # k8s/bases/infrastructure/vault-seed/). Until then, @@ -412,7 +425,7 @@ kubectl -n kube-system get cm cluster-autoscaler-status -o yaml Common causes: - Pool `maxSize` reached — increase `max` under the relevant pool in `ksail.prod.yaml`, then run - `ksail --config ksail.prod.yaml cluster update` + `./scripts/run-ksail-prod-with-pull-auth.sh cluster update` - `HCLOUD_TOKEN` expired — rotate in SOPS secrets and GitHub environment secrets ### Orphaned autoscaler nodes after cluster delete @@ -435,7 +448,7 @@ hcloud server list # If the server exists but node doesn't appear in kubectl: # The worker machine config may be invalid or stale. # Re-run cluster update to regenerate worker config and re-apply: -ksail --config ksail.prod.yaml cluster update +./scripts/run-ksail-prod-with-pull-auth.sh cluster update ``` --- @@ -490,17 +503,17 @@ gh secret set TALOS_CONFIG --env prod --repo devantler-tech/platform < ~/.talos/ > step in the deploy workflows is the local guard against that behaviour until > those land. -## Scenario 10 — Flux/tenant GHCR pull credential rotation or denial +## Scenario 10 — production GHCR pull credential rotation or denial -The authoritative Flux and tenant pull credential is the SOPS-encrypted +The authoritative Flux, tenant, Kyverno, and Talos-host pull credential is the +SOPS-encrypted `stringData.ghcr_dockerconfigjson` value in `k8s/bases/bootstrap/secret.enc.yaml`. It is deliberately separate from the -GitHub `prod` environment's `GHCR_TOKEN`, which pushes/signs the artifact and is -still passed to KSail/Talos for compatibility. - -Talos node registry auth still derives from `GHCR_TOKEN`; this recovery path -does not rotate existing node machine configuration. That remaining -single-source consolidation stays tracked by #2613. +GitHub `prod` environment's `GHCR_TOKEN`, which only pushes/signs artifacts. +Production KSail lifecycle commands run through +`scripts/run-ksail-prod-with-pull-auth.sh`, which derives username/token from +Git/SOPS and supplies a non-secret SOPS-ciphertext revision to the Talos +machine/autoscaler template. The production deploy closes the bootstrap loop in this order: @@ -509,7 +522,14 @@ The production deploy closes the bootstrap loop in this order: artifacts, both tenant application images, and the KSail plus provider-upjet-unifi packages used by Kyverno verification. During DR, a read-only `--check-only` pass happens before infrastructure creation. -2. Before changing mutable `latest`, patch `variables-base`; force-sync +2. Before changing mutable `latest`, list every Kubernetes Node (including + NotReady/autoscaled nodes). For each node whose **verified** ciphertext + revision is stale, apply the supported Talos `RegistryAuthConfig` in + no-reboot mode, workers before control planes; pull the exact live KSail + operator image through the Talos CRI; and only then record the verified + revision. A distinct desired revision in the committed Talos configuration + refreshes future Cluster Autoscaler templates but never counts as proof. +3. Patch `variables-base`; force-sync `seed-ghcr` into OpenBao; force-sync the tenant/Kyverno ExternalSecrets; verify every materialised `ghcr-auth` payload matches Git/SOPS; and only then patch `flux-system/ksail-registry-credentials`. A fresh or partial DR @@ -517,14 +537,15 @@ The production deploy closes the bootstrap loop in this order: `--allow-incomplete-fanout` mode stages `variables-base` and repairs root auth so the first reconcile can create the chain. Normal mode fails closed on any missing fan-out resource. -3. Push and sign the artifact with `GHCR_TOKEN`, revalidate the newly-published +4. Push and sign the artifact with `GHCR_TOKEN`, revalidate the newly-published artifact with `--check-only`, and only then explicitly reconcile Flux. DR runs the full bridge again after every Flux Kustomization is Ready, proving that bootstrap mode completed the entire fan-out, and once more after an OpenBao raft restore so a snapshot cannot rematerialise an older credential. -4. Re-run the bridge after `cluster update`, because KSail can rewrite its - managed root Secret when another cluster setting changes; the bridge also - re-verifies the downstream fan-out. +5. Re-run the bridge after `cluster update`; it repairs any node left stale by + a partial lifecycle operation and re-verifies the root plus downstream + fan-out. Nodes already carrying the verified revision skip Talos API calls, + so an ordinary deploy does not depend on every Talos endpoint. For a normal rotation, update the encrypted value through a PR and let the merge queue deploy it. If the encrypted file was pushed directly to `main`, manually diff --git a/ksail.prod.yaml b/ksail.prod.yaml index fb8373168..04d7466e6 100644 --- a/ksail.prod.yaml +++ b/ksail.prod.yaml @@ -137,7 +137,7 @@ spec: - siderolabs/util-linux-tools - siderolabs/qemu-guest-agent localRegistry: - registry: "devantler:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests" + registry: "${GHCR_USERNAME}:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests" # Cosign keyless signature verification on the flux-system OCIRepository that # KSail generates and owns (rendered onto spec.verify; a hand-written override # would lose the reconcile fight, so it is configured here per ksail#4987). diff --git a/scripts/ghcr-auth-lib.sh b/scripts/ghcr-auth-lib.sh new file mode 100644 index 000000000..ecc3e47b5 --- /dev/null +++ b/scripts/ghcr-auth-lib.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Shared secret-safe helpers for the Git/SOPS GHCR pull credential. + +# Decrypt only the Docker config scalar into a caller-owned restricted file. +decrypt_flux_ghcr_docker_config() { + local output_file="$1" + local secret_file="${2:-k8s/bases/bootstrap/secret.enc.yaml}" + + ksail workload cipher decrypt \ + "${secret_file}" \ + --extract '["stringData"]["ghcr_dockerconfigjson"]' \ + --output "${output_file}" \ + >/dev/null + chmod 600 "${output_file}" +} + +# Validate the Docker config and write canonical username/password JSON. +write_flux_ghcr_credentials() { + local docker_config="$1" + local output_file="$2" + + if ! jq -e ' + def non_empty_string: type == "string" and length > 0; + (.auths["ghcr.io"] // {}) as $auth + | ((($auth | has("username")) or ($auth | has("password")))) + as $explicit_present + | (($auth.username | non_empty_string) + and ($auth.password | non_empty_string)) as $explicit_valid + | ($auth | has("auth")) as $encoded_present + | (if $encoded_present then + try ( + $auth.auth + | @base64d + | capture("^(?[^:]+):(?.+)$") + ) catch null + else null end) as $decoded + | (($decoded != null) + and ($decoded.username | non_empty_string) + and ($decoded.password | non_empty_string)) as $encoded_valid + | select( + ((($explicit_present | not) or $explicit_valid) + and (($encoded_present | not) or $encoded_valid) + and ($explicit_valid or $encoded_valid) + and (((($explicit_present and $encoded_present) | not)) + or (($auth.username == $decoded.username) + and ($auth.password == $decoded.password)))) + ) + | if $explicit_valid then + {username: $auth.username, password: $auth.password} + else + {username: $decoded.username, password: $decoded.password} + end + ' "${docker_config}" > "${output_file}"; then + echo "::error::The SOPS GHCR pull credential is not a valid Docker config with non-empty, consistent ghcr.io username/password and auth fields." + return 1 + fi + chmod 600 "${output_file}" +} + +# Print a non-secret ciphertext revision for redaction-resistant drift detection. +flux_ghcr_revision() { + local secret_file="${1:-k8s/bases/bootstrap/secret.enc.yaml}" + + # Hash the committed SOPS ciphertext, not the decrypted credential. This + # changes when the pull credential is rotated without publishing a stable + # verifier for the token itself in the Kubernetes Node annotation. + yq -er ' + .stringData.ghcr_dockerconfigjson + | select(tag == "!!str" and length > 0 and test("^ENC\\[")) + ' "${secret_file}" \ + | shasum -a 256 \ + | awk '{print $1}' +} diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index b6069c6c2..c8e8a3b7a 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -6,6 +6,12 @@ # repair that bootstrap edge before asking Flux to reconcile. set -euo pipefail +set +x + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +# shellcheck source=scripts/ghcr-auth-lib.sh +source "${SCRIPT_DIR}/ghcr-auth-lib.sh" check_only=false allow_incomplete_fanout=false @@ -28,13 +34,16 @@ readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.ya readonly KUBE_CONTEXT="${KUBE_CONTEXT:-admin@prod}" readonly SYNC_ATTEMPTS="${FLUX_GHCR_SYNC_ATTEMPTS:-60}" readonly SYNC_INTERVAL="${FLUX_GHCR_SYNC_INTERVAL:-2}" +KSAIL_OPERATOR_VERSION="$(yq -er '.spec.chart.spec.version' \ + k8s/bases/infrastructure/controllers/ksail-operator/helm-release.yaml)" +readonly KSAIL_OPERATOR_VERSION readonly -a REQUIRED_PULL_TARGETS=( "devantler-tech/platform/manifests:latest" "devantler-tech/wedding-app/manifests:latest" "devantler-tech/ascoachingogvaner/manifests:latest" "devantler-tech/wedding-app:latest" "devantler-tech/ascoachingogvaner:latest" - "devantler-tech/ksail:latest" + "devantler-tech/ksail:v${KSAIL_OPERATOR_VERSION}" "devantler-tech/provider-upjet-unifi:v0.1.0" ) readonly -a FANOUT_NAMESPACES=( @@ -55,6 +64,7 @@ chmod 700 "${work_dir}" umask 077 docker_config="${work_dir}/config.json" +credentials_file="${work_dir}/credentials.json" basic_curl_config="${work_dir}/curl-basic.config" bearer_curl_config="${work_dir}/curl-bearer.config" token_response="${work_dir}/token.json" @@ -62,6 +72,12 @@ patch_file="${work_dir}/patch.json" variables_patch_file="${work_dir}/variables-patch.json" expected_normalized="${work_dir}/expected-normalized.json" fanout_api_resources="${work_dir}/fanout-api-resources.txt" +talos_auth_patch_file="${work_dir}/talos-registry-auth.json" +talos_revision_patch_file="${work_dir}/talos-registry-revision.json" +talos_result_file="${work_dir}/talos-result.txt" +talos_nodes_file="${work_dir}/talos-nodes.json" +talos_node_targets="${work_dir}/talos-node-targets.tsv" +ksail_operator_deployment="${work_dir}/ksail-operator-deployment.json" # Force an ESO resource to reconcile and observe a post-annotation Ready edge. force_sync_resource() { @@ -145,59 +161,151 @@ verify_consumer_secret() { fi } +# Apply Git/SOPS auth to stale Talos nodes, prove an exact image pull, and only +# then record the non-secret revision marker that makes the operation retryable. +sync_talos_registry_auth() { + local desired_revision="$1" + local operator_image="ghcr.io/devantler-tech/ksail:v${KSAIL_OPERATOR_VERSION}" + local image_reference + local node_name + local node_ip + local _node_role + + if ! kubectl \ + --context "${KUBE_CONTEXT}" \ + get nodes \ + -o json \ + > "${talos_nodes_file}"; then + echo "::error::Could not list Talos nodes; refusing to mutate any Kubernetes credential consumers." + return 1 + fi + if ! jq -e ' + (.items | length) > 0 + and all(.items[]; + ([.status.addresses[]? | select(.type == "InternalIP") | .address] + | length) == 1 + and (([.status.addresses[]? + | select(.type == "InternalIP") | .address][0]) + | type == "string" and length > 0)) + and (([.items[] + | [.status.addresses[]? + | select(.type == "InternalIP") | .address][0]] + | unique | length) == (.items | length)) + ' "${talos_nodes_file}" >/dev/null; then + echo "::error::Every Talos node must expose exactly one non-empty, unique InternalIP before GHCR auth can be synchronized." + return 1 + fi + + if ! jq -r --arg revision "${desired_revision}" ' + .items[] + | select( + (.metadata.annotations[ + "platform.devantler.tech/ghcr-pull-verified-revision" + ] // "") + != $revision + ) + | (.metadata.labels // {}) as $labels + | [ + (if (($labels | has("node-role.kubernetes.io/control-plane")) + or ($labels | has("node-role.kubernetes.io/master"))) + then "1" else "0" end), + .metadata.name, + ([.status.addresses[] + | select(.type == "InternalIP") | .address][0]) + ] + | @tsv + ' "${talos_nodes_file}" \ + | LC_ALL=C sort -k1,1 -k2,2 \ + > "${talos_node_targets}"; then + echo "::error::Could not select Talos nodes requiring the GHCR auth revision." + return 1 + fi + + # Normal deploys should not regain an all-node Talos API dependency once the + # current ciphertext revision has been proved on every node. + if [[ ! -s "${talos_node_targets}" ]]; then + return 0 + fi + + if ! kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace ksail-operator \ + get deployment ksail-operator \ + --ignore-not-found \ + -o json \ + > "${ksail_operator_deployment}"; then + echo "::error::Could not determine the exact live KSail operator image." + return 1 + fi + if [[ -s "${ksail_operator_deployment}" ]]; then + if ! operator_image="$(jq -er ' + [.spec.template.spec.containers[]? | select(.name == "operator") | .image] + | if length == 1 then .[0] else error("expected one operator image") end + ' "${ksail_operator_deployment}")"; then + echo "::error::The live ksail-operator Deployment does not expose one expected first-party image." + return 1 + fi + elif [[ "${allow_incomplete_fanout}" != "true" ]]; then + echo "::error::The live ksail-operator Deployment is absent outside explicit DR bootstrap mode." + return 1 + fi + + if [[ "${operator_image}" == ghcr.io/devantler-tech/ksail:* ]]; then + image_reference="${operator_image##*:}" + if [[ -z "${image_reference}" || "${image_reference}" == "latest" ]]; then + echo "::error::The KSail operator verification image must use an exact non-latest tag or digest." + return 1 + fi + elif ! [[ "${operator_image}" =~ ^ghcr\.io/devantler-tech/ksail@sha256:[0-9a-f]{64}$ ]]; then + echo "::error::The KSail operator verification image must be the expected first-party package." + return 1 + fi + + : > "${talos_result_file}" + chmod 600 "${talos_result_file}" + while IFS=$'\t' read -r _node_role node_name node_ip; do + if ! talosctl \ + --nodes "${node_ip}" \ + patch machineconfig \ + --mode=no-reboot \ + --patch-file="${talos_auth_patch_file}" \ + >"${talos_result_file}" 2>&1; then + echo "::error::Talos node ${node_name} did not accept the Git/SOPS GHCR registry auth." + return 1 + fi + if ! talosctl \ + --nodes "${node_ip}" \ + image pull "${operator_image}" \ + --namespace cri \ + >"${talos_result_file}" 2>&1; then + echo "::error::Talos node ${node_name} could not pull the exact live KSail image after its auth refresh." + return 1 + fi + if ! talosctl \ + --nodes "${node_ip}" \ + patch machineconfig \ + --mode=no-reboot \ + --patch-file="${talos_revision_patch_file}" \ + >"${talos_result_file}" 2>&1; then + echo "::error::Talos node ${node_name} proved GHCR access but could not record the synchronized credential revision." + return 1 + fi + done < "${talos_node_targets}" +} + # KSail embeds SOPS, so the deploy uses the same pinned toolchain as workload # reconciliation. Decrypt only the Docker config scalar and never emit it to # stdout or place its plaintext/base64 representation in an argument. -ksail workload cipher decrypt \ - "${SECRET_FILE}" \ - --extract '["stringData"]["ghcr_dockerconfigjson"]' \ - --output "${docker_config}" \ - >/dev/null -chmod 600 "${docker_config}" - -if ! jq -e ' - def non_empty_string: type == "string" and length > 0; - (.auths["ghcr.io"] // {}) as $auth - | ((($auth | has("username")) or ($auth | has("password")))) - as $explicit_present - | (($auth.username | non_empty_string) - and ($auth.password | non_empty_string)) as $explicit_valid - | ($auth | has("auth")) as $encoded_present - | (if $encoded_present then - try ( - $auth.auth - | @base64d - | capture("^(?[^:]+):(?.+)$") - ) catch null - else null end) as $decoded - | (($decoded != null) - and ($decoded.username | non_empty_string) - and ($decoded.password | non_empty_string)) as $encoded_valid - | ((($explicit_present | not) or $explicit_valid) - and (($encoded_present | not) or $encoded_valid) - and ($explicit_valid or $encoded_valid) - and (((($explicit_present and $encoded_present) | not)) - or (($auth.username == $decoded.username) - and ($auth.password == $decoded.password)))) -' "${docker_config}" >/dev/null; then - echo "::error::The SOPS GHCR pull credential is not a valid Docker config with non-empty, consistent ghcr.io username/password and auth fields." - exit 1 -fi +decrypt_flux_ghcr_docker_config "${docker_config}" "${SECRET_FILE}" +write_flux_ghcr_credentials "${docker_config}" "${credentials_file}" jq -S -c . "${docker_config}" > "${expected_normalized}" # Build curl's Basic-auth config without putting the credential in argv or # stdout. Support both Docker config representations used in this repository: # explicit username/password and base64(username:password) in auth. jq -r ' - def non_empty_string: type == "string" and length > 0; - (.auths["ghcr.io"] // {}) as $auth - | if (($auth.username | non_empty_string) - and ($auth.password | non_empty_string)) then - "user = " + (($auth.username + ":" + $auth.password) | @json) - else - "user = " + (($auth.auth | @base64d) | @json) - end -' "${docker_config}" > "${basic_curl_config}" + "user = " + ((.username + ":" + .password) | @json) +' "${credentials_file}" > "${basic_curl_config}" chmod 600 "${basic_curl_config}" # The same pull credential fans out to Flux OCI sources and private tenant @@ -259,6 +367,33 @@ if [[ "${check_only}" == "true" ]]; then exit 0 fi +# Talos image verification resolves cosign artifacts with host registry auth; +# pod imagePullSecrets cannot satisfy that request. Use the supported v1.13 +# RegistryAuthConfig document and a file-backed patch so credentials never +# enter argv. Synchronize and verify nodes before touching Kubernetes Secrets. +jq ' + { + apiVersion: "v1alpha1", + kind: "RegistryAuthConfig", + name: "ghcr.io", + username: .username, + password: .password + } +' "${credentials_file}" > "${talos_auth_patch_file}" +pull_revision="$(flux_ghcr_revision "${SECRET_FILE}")" +readonly pull_revision +jq -n --arg revision "${pull_revision}" ' + { + machine: { + nodeAnnotations: { + "platform.devantler.tech/ghcr-pull-verified-revision": $revision + } + } + } +' > "${talos_revision_patch_file}" +chmod 600 "${talos_auth_patch_file}" "${talos_revision_patch_file}" +sync_talos_registry_auth "${pull_revision}" + # Merge only Secret data fields so ownership metadata survives. The sensitive # payload stays in pipes/temp files and never appears in argv or logs. base64 < "${docker_config}" \ diff --git a/scripts/run-ksail-prod-with-pull-auth.sh b/scripts/run-ksail-prod-with-pull-auth.sh new file mode 100755 index 000000000..2a558f535 --- /dev/null +++ b/scripts/run-ksail-prod-with-pull-auth.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Run credential-consuming KSail lifecycle commands with Git/SOPS pull auth. + +set -euo pipefail +set +x + +case "${1:-} ${2:-}" in + "cluster create" | "cluster update" | "workload push" | "workload reconcile") ;; + *) + echo "Usage: $0 {cluster create|cluster update|workload push|workload reconcile}" >&2 + exit 64 + ;; +esac +if (($# != 2)); then + echo "Usage: $0 {cluster create|cluster update|workload push|workload reconcile}" >&2 + exit 64 +fi + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +# shellcheck source=scripts/ghcr-auth-lib.sh +source "${SCRIPT_DIR}/ghcr-auth-lib.sh" + +readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" +pull_revision="$(flux_ghcr_revision "${SECRET_FILE}")" +readonly pull_revision + +# A shell variable is required to pass the token in the child environment. +# It never enters argv or stdout, and the constrained command set prevents an +# arbitrary process from inheriting it. The Actions GHCR_TOKEN remains in the +# workload-push step; only lifecycle reads override it. The non-secret SOPS +# ciphertext revision defeats KSail's deliberate credential redaction in +# machine-config fingerprints, so autoscaler templates still refresh on +# token-only rotations. +if [[ "$1 $2" == "workload push" ]]; then + if [[ -z "${GHCR_TOKEN:-}" || -z "${GITHUB_ACTOR:-}" ]]; then + echo "::error::The Actions GHCR publish token and actor are required for workload push." + exit 1 + fi + GHCR_USERNAME="${GITHUB_ACTOR}" \ + GHCR_PULL_REVISION="${pull_revision}" \ + ksail --config ksail.prod.yaml "$1" "$2" +else + work_dir="$(mktemp -d)" + trap 'rm -rf "${work_dir}"' EXIT + chmod 700 "${work_dir}" + umask 077 + + docker_config="${work_dir}/config.json" + credentials_file="${work_dir}/credentials.json" + decrypt_flux_ghcr_docker_config "${docker_config}" "${SECRET_FILE}" + write_flux_ghcr_credentials "${docker_config}" "${credentials_file}" + pull_username="$(jq -er '.username' "${credentials_file}")" + pull_token="$(jq -er '.password' "${credentials_file}")" + GHCR_USERNAME="${pull_username}" \ + GHCR_TOKEN="${pull_token}" \ + GHCR_PULL_REVISION="${pull_revision}" \ + ksail --config ksail.prod.yaml "$1" "$2" +fi diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index 6b24f1c6e..8be306bf1 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import hashlib import json import os from pathlib import Path @@ -14,8 +15,18 @@ ROOT = Path(__file__).resolve().parents[2] HELPER = ROOT / "scripts" / "refresh-flux-ghcr-auth.sh" +KSAIL_PULL_WRAPPER = ROOT / "scripts" / "run-ksail-prod-with-pull-auth.sh" ACTION = ROOT / ".github" / "actions" / "deploy-prod" / "action.yml" DR_REBUILD = ROOT / ".github" / "workflows" / "dr-rebuild.yaml" +KSAIL_OPERATOR_HELM_RELEASE = ( + ROOT + / "k8s" + / "bases" + / "infrastructure" + / "controllers" + / "ksail-operator" + / "helm-release.yaml" +) PROVIDER_UPJET_UNIFI = ( ROOT / "k8s" @@ -25,6 +36,10 @@ / "crossplane" / "provider-upjet-unifi.yaml" ) +TALOS_GHCR_AUTH = ROOT / "talos" / "cluster" / "authenticate-ghcr-pulls.yaml" +TALOS_GHCR_REVISION = ( + ROOT / "talos" / "cluster" / "mark-ghcr-pull-revision.yaml" +) class RefreshFluxGhcrAuthTests(unittest.TestCase): @@ -38,14 +53,23 @@ def setUp(self) -> None: self.bin_dir = self.workspace / "bin" self.bin_dir.mkdir() self.decrypted_config = self.workspace / "decrypted-config.json" + self.encrypted_secret = self.workspace / "secret.enc.yaml" self.patch_capture = self.workspace / "patch.json" self.variables_patch_capture = self.workspace / "variables-patch.json" self.kubectl_called = self.workspace / "kubectl-called" self.output_path_log = self.workspace / "ksail-output-path" self.registry_read_log = self.workspace / "registry-reads" self.fanout_log = self.workspace / "fanout-log" + self.talos_log = self.workspace / "talos-log" + self.talos_patch_path_log = self.workspace / "talos-patch-path" + self.operation_log = self.workspace / "operation-log" + self.ksail_token_capture = self.workspace / "ksail-token" + self.ksail_username_capture = self.workspace / "ksail-username" + self.ksail_revision_capture = self.workspace / "ksail-revision" + self.ksail_command_capture = self.workspace / "ksail-command" self.sync_state_dir = self.workspace / "sync-state" self.sync_state_dir.mkdir() + self._write_encrypted_secret("ENC[AES256_GCM,data:fixture-one]") self._write_executable( "ksail", @@ -53,23 +77,133 @@ def setUp(self) -> None: #!/usr/bin/env bash set -euo pipefail arguments=" $* " - secret_file='k8s/bases/bootstrap/secret.enc.yaml' selector='["stringData"]["ghcr_dockerconfigjson"]' - [[ "$arguments" == *" workload cipher decrypt $secret_file "* ]] - [[ "$arguments" == *" --extract $selector "* ]] - output="" - while (($#)); do - case "$1" in - --output) - output="$2" - shift 2 - ;; - *) shift ;; + if [[ "$arguments" == *" workload cipher decrypt "* ]]; then + [[ "$arguments" == *" --extract $selector "* ]] + output="" + while (($#)); do + case "$1" in + --output) + output="$2" + shift 2 + ;; + *) shift ;; + esac + done + test -n "$output" + printf '%s' "$output" > "$KSAIL_OUTPUT_PATH_LOG" + cp "$FAKE_DECRYPTED_CONFIG" "$output" + exit 0 + fi + + if [[ "$arguments" == *" --config ksail.prod.yaml cluster create "* \ + || "$arguments" == *" --config ksail.prod.yaml cluster update "* \ + || "$arguments" == *" --config ksail.prod.yaml workload push "* \ + || "$arguments" == *" --config ksail.prod.yaml workload reconcile "* ]]; then + test -n "${GHCR_TOKEN:-}" + test -n "${GHCR_USERNAME:-}" + test "${GHCR_PULL_REVISION:-}" != "" + printf '%s' "$GHCR_TOKEN" > "$KSAIL_TOKEN_CAPTURE" + printf '%s' "$GHCR_USERNAME" > "$KSAIL_USERNAME_CAPTURE" + printf '%s' "$GHCR_PULL_REVISION" > "$KSAIL_REVISION_CAPTURE" + printf '%s\n' "$*" > "$KSAIL_COMMAND_CAPTURE" + exit 0 + fi + + echo "unexpected ksail invocation" >&2 + exit 92 + """, + ) + self._write_executable( + "talosctl", + """ + #!/usr/bin/env bash + set -euo pipefail + arguments=" $* " + node="" + patch_file="" + previous="" + for argument in "$@"; do + if [[ "$previous" == --nodes ]]; then + node="$argument" + fi + case "$argument" in + --nodes=*) node="${argument#*=}" ;; + --patch-file=*) patch_file="${argument#*=}" ;; esac + previous="$argument" done - test -n "$output" - printf '%s' "$output" > "$KSAIL_OUTPUT_PATH_LOG" - cp "$FAKE_DECRYPTED_CONFIG" "$output" + test -n "$node" + + if [[ "$arguments" == *" patch machineconfig "* ]]; then + [[ "$arguments" == *" --mode=no-reboot "* ]] + test -f "$patch_file" + printf '%s' "$patch_file" > "$TALOS_PATCH_PATH_LOG" + if jq -e '.kind == "RegistryAuthConfig"' "$patch_file" >/dev/null; then + jq -e \ + --arg username "$EXPECTED_PULL_USERNAME" \ + --arg token "$EXPECTED_PULL_TOKEN" ' + .apiVersion == "v1alpha1" + and .kind == "RegistryAuthConfig" + and .name == "ghcr.io" + and .username == $username + and .password == $token + ' "$patch_file" >/dev/null + printf 'talos-auth:%s\n' "$node" >> "$TALOS_LOG" + printf 'talos-auth:%s\n' "$node" >> "$OPERATION_LOG" + if [[ "$node" == "${FAKE_TALOS_FAIL_NODE:-disabled}" \ + && "${FAKE_TALOS_FAIL_OPERATION:-auth}" == auth ]]; then + echo "talos auth failed with $EXPECTED_PULL_TOKEN" >&2 + exit 45 + fi + touch "$FAKE_SYNC_STATE_DIR/talos-auth-${node}" + exit 0 + fi + + test -f "$FAKE_SYNC_STATE_DIR/talos-auth-${node}" + test -f "$FAKE_SYNC_STATE_DIR/talos-pull-${node}" + jq -e --arg revision "$EXPECTED_GHCR_REVISION" ' + .machine.nodeAnnotations[ + "platform.devantler.tech/ghcr-pull-verified-revision" + ] == $revision + ' "$patch_file" >/dev/null + printf 'talos-revision:%s\n' "$node" >> "$TALOS_LOG" + printf 'talos-revision:%s\n' "$node" >> "$OPERATION_LOG" + if [[ "$node" == "${FAKE_TALOS_FAIL_NODE:-disabled}" \ + && "${FAKE_TALOS_FAIL_OPERATION:-auth}" == revision ]]; then + echo "talos revision failed" >&2 + exit 48 + fi + touch "$FAKE_SYNC_STATE_DIR/talos-revision-${node}" + exit 0 + fi + + if [[ "$arguments" == *" image pull "* ]]; then + test -f "$FAKE_SYNC_STATE_DIR/talos-auth-${node}" + [[ "$arguments" == *" --namespace cri "* ]] + image="" + previous="" + for argument in "$@"; do + if [[ "$previous" == pull ]]; then + image="$argument" + break + fi + previous="$argument" + done + test -n "$image" + printf 'talos-pull:%s:%s\n' "$node" "$image" >> "$TALOS_LOG" + printf 'talos-pull:%s:%s\n' "$node" "$image" >> "$OPERATION_LOG" + if [[ "$node" == "${FAKE_TALOS_FAIL_NODE:-disabled}" \ + && "${FAKE_TALOS_FAIL_OPERATION:-auth}" == pull ]]; then + echo "talos pull failed with $EXPECTED_PULL_TOKEN" >&2 + exit 47 + fi + touch "$FAKE_SYNC_STATE_DIR/talos-pull-${node}" + exit 0 + fi + + echo "unexpected talosctl invocation" >&2 + exit 93 """, ) self._write_executable( @@ -159,9 +293,73 @@ def setUp(self) -> None: esac previous="$argument" done - test -n "$namespace" touch "$KUBECTL_CALLED" + if [[ "$arguments" == *" get nodes "* ]]; then + if [[ "${FAKE_NODE_DISCOVERY_FAIL:-false}" == true ]]; then + echo 'node discovery failed' >&2 + exit 46 + fi + if [[ -n "${FAKE_NODE_JSON:-}" ]]; then + printf '%s\n' "$FAKE_NODE_JSON" + exit 0 + fi + jq -n \ + --arg revision "$EXPECTED_GHCR_REVISION" \ + --arg current "${FAKE_TALOS_NODES_CURRENT:-false}" ' + { + items: [ + { + metadata: { + name: "prod-worker-1", + labels: {}, + annotations: { + "platform.devantler.tech/ghcr-pull-desired-revision": + $revision + } + }, + status: {addresses: [ + {type: "InternalIP", address: "10.0.0.2"} + ]} + }, + { + metadata: { + name: "prod-control-plane-1", + labels: { + "node-role.kubernetes.io/control-plane": "" + }, + annotations: { + "platform.devantler.tech/ghcr-pull-desired-revision": + $revision + } + }, + status: {addresses: [ + {type: "InternalIP", address: "10.0.0.1"} + ]} + } + ] + } + | if $current == "true" then + .items[].metadata.annotations[ + "platform.devantler.tech/ghcr-pull-verified-revision" + ] = $revision + else . end + ' + exit 0 + fi + + test -n "$namespace" + + if [[ "$arguments" == *" get deployment ksail-operator "* ]]; then + test "$namespace" = ksail-operator + if [[ "${FAKE_KSAIL_OPERATOR_DEPLOYMENT_ABSENT:-false}" == true ]]; then + exit 0 + fi + printf '{"spec":{"template":{"spec":{"containers":[{"name":"sidecar","image":"public.example/sidecar:latest"},{"name":"operator","image":"%s"}]}}}}\n' \ + "${FAKE_KSAIL_OPERATOR_IMAGE:-ghcr.io/devantler-tech/ksail:v7.169.0}" + exit 0 + fi + if [[ "$arguments" == *" api-resources "* ]]; then [[ "$arguments" == *" --api-group=external-secrets.io "* ]] if [[ "${FAKE_FANOUT_CRDS_ABSENT:-false}" != true ]]; then @@ -176,6 +374,7 @@ def setUp(self) -> None: [[ "$arguments" == *" --type=merge "* ]] test -n "$patch_file" cp "$patch_file" "$PATCH_CAPTURE" + printf 'root-patch\n' >> "$OPERATION_LOG" if [[ "${FAKE_KUBECTL_FAIL:-false}" == true ]]; then echo 'cluster patch failed' >&2 exit 43 @@ -196,6 +395,7 @@ def setUp(self) -> None: [[ "$arguments" == *" --type=merge "* ]] test -n "$patch_file" cp "$patch_file" "$VARIABLES_PATCH_CAPTURE" + printf 'variables-patch\n' >> "$OPERATION_LOG" echo 'secret/variables-base patched' exit 0 fi @@ -246,6 +446,7 @@ def setUp(self) -> None: if [[ -n "$kind" && "$arguments" == *" annotate $kind $name "* ]]; then resource="$kind/$namespace/$name" printf '%s\n' "$resource" >> "$FANOUT_LOG" + printf 'fanout:%s\n' "$resource" >> "$OPERATION_LOG" marker="$FAKE_SYNC_STATE_DIR/${kind}-${namespace}-${name}" touch "${marker}-annotated" if [[ "$resource" != "${FAKE_SYNC_STALL_RESOURCE:-disabled}" ]]; then @@ -276,6 +477,37 @@ def _write_executable(self, name: str, body: str) -> None: path.write_text(textwrap.dedent(body).lstrip(), encoding="utf-8") path.chmod(0o755) + def _write_encrypted_secret(self, ciphertext: str) -> None: + """Write a non-secret SOPS ciphertext fixture for revision tests.""" + self.encrypted_ciphertext = ciphertext + self.encrypted_secret.write_text( + json.dumps( + {"stringData": {"ghcr_dockerconfigjson": ciphertext}} + ), + encoding="utf-8", + ) + + @staticmethod + def _expected_credentials(config: object) -> tuple[str, str]: + """Extract expected credentials from a valid Docker config fixture.""" + try: + registry = config["auths"]["ghcr.io"] # type: ignore[index] + username = registry.get("username") + password = registry.get("password") + if username and password: + return str(username), str(password) + decoded = base64.b64decode(registry["auth"]).decode() + decoded_username, decoded_password = decoded.split(":", 1) + return decoded_username, decoded_password + except (KeyError, TypeError, ValueError): + return "unused", "unused" + + def _expected_revision(self) -> str: + """Match the helper's SHA-256 of the yq-emitted ciphertext scalar.""" + return hashlib.sha256( + f"{self.encrypted_ciphertext}\n".encode() + ).hexdigest() + def _run_helper( self, config: object, @@ -291,21 +523,40 @@ def _run_helper( self.output_path_log, self.registry_read_log, self.fanout_log, + self.talos_log, + self.talos_patch_path_log, + self.operation_log, + self.ksail_token_capture, + self.ksail_username_capture, + self.ksail_revision_capture, + self.ksail_command_capture, ): marker.unlink(missing_ok=True) for marker in self.sync_state_dir.iterdir(): marker.unlink() + expected_username, expected_token = self._expected_credentials(config) environment = os.environ.copy() environment.update( { "PATH": f"{self.bin_dir}:{environment['PATH']}", "FAKE_DECRYPTED_CONFIG": str(self.decrypted_config), + "FLUX_GHCR_SECRET_FILE": str(self.encrypted_secret), "PATCH_CAPTURE": str(self.patch_capture), "VARIABLES_PATCH_CAPTURE": str(self.variables_patch_capture), "KUBECTL_CALLED": str(self.kubectl_called), "KSAIL_OUTPUT_PATH_LOG": str(self.output_path_log), "REGISTRY_READ_LOG": str(self.registry_read_log), "FANOUT_LOG": str(self.fanout_log), + "TALOS_LOG": str(self.talos_log), + "TALOS_PATCH_PATH_LOG": str(self.talos_patch_path_log), + "OPERATION_LOG": str(self.operation_log), + "KSAIL_TOKEN_CAPTURE": str(self.ksail_token_capture), + "KSAIL_USERNAME_CAPTURE": str(self.ksail_username_capture), + "KSAIL_REVISION_CAPTURE": str(self.ksail_revision_capture), + "KSAIL_COMMAND_CAPTURE": str(self.ksail_command_capture), + "EXPECTED_PULL_USERNAME": expected_username, + "EXPECTED_PULL_TOKEN": expected_token, + "EXPECTED_GHCR_REVISION": self._expected_revision(), "FAKE_SYNC_STATE_DIR": str(self.sync_state_dir), "FLUX_GHCR_SYNC_ATTEMPTS": "2", "FLUX_GHCR_SYNC_INTERVAL": "0", @@ -321,6 +572,49 @@ def _run_helper( check=False, ) + def _run_ksail_pull_wrapper( + self, + config: object, + command: tuple[str, ...], + **environment_overrides: str, + ) -> subprocess.CompletedProcess[str]: + """Run a production KSail command through the SOPS pull-auth wrapper.""" + self.decrypted_config.write_text(json.dumps(config), encoding="utf-8") + for marker in ( + self.output_path_log, + self.ksail_token_capture, + self.ksail_username_capture, + self.ksail_revision_capture, + self.ksail_command_capture, + ): + marker.unlink(missing_ok=True) + expected_username, expected_token = self._expected_credentials(config) + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.bin_dir}:{environment['PATH']}", + "FAKE_DECRYPTED_CONFIG": str(self.decrypted_config), + "FLUX_GHCR_SECRET_FILE": str(self.encrypted_secret), + "KSAIL_OUTPUT_PATH_LOG": str(self.output_path_log), + "KSAIL_TOKEN_CAPTURE": str(self.ksail_token_capture), + "KSAIL_USERNAME_CAPTURE": str(self.ksail_username_capture), + "KSAIL_REVISION_CAPTURE": str(self.ksail_revision_capture), + "KSAIL_COMMAND_CAPTURE": str(self.ksail_command_capture), + "EXPECTED_PULL_USERNAME": expected_username, + "EXPECTED_PULL_TOKEN": expected_token, + "EXPECTED_GHCR_REVISION": self._expected_revision(), + } + ) + environment.update(environment_overrides) + return subprocess.run( + [str(KSAIL_PULL_WRAPPER), *command], + cwd=ROOT, + env=environment, + text=True, + capture_output=True, + check=False, + ) + @staticmethod def _valid_config() -> dict[str, object]: """Return a valid explicit GHCR Docker authentication fixture.""" @@ -360,7 +654,7 @@ def test_refreshes_root_and_fanout_without_leaking_plaintext(self) -> None: "devantler-tech/ascoachingogvaner/manifests:latest", "devantler-tech/wedding-app:latest", "devantler-tech/ascoachingogvaner:latest", - "devantler-tech/ksail:latest", + "devantler-tech/ksail:v7.167.0", "devantler-tech/provider-upjet-unifi:v0.1.0", ], ) @@ -374,6 +668,281 @@ def test_refreshes_root_and_fanout_without_leaking_plaintext(self) -> None: ], ) + def test_refreshes_talos_auth_before_kubernetes_credential_consumers(self) -> None: + """Patch nodes serially and pull the exact live operator image first.""" + result = self._run_helper(self._valid_config()) + + self.assertEqual(result.returncode, 0, result.stderr) + expected_talos_operations = [ + "talos-auth:10.0.0.2", + "talos-pull:10.0.0.2:ghcr.io/devantler-tech/ksail:v7.169.0", + "talos-revision:10.0.0.2", + "talos-auth:10.0.0.1", + "talos-pull:10.0.0.1:ghcr.io/devantler-tech/ksail:v7.169.0", + "talos-revision:10.0.0.1", + ] + self.assertEqual( + self.talos_log.read_text(encoding="utf-8").splitlines(), + expected_talos_operations, + ) + self.assertEqual( + self.operation_log.read_text(encoding="utf-8").splitlines()[:6], + expected_talos_operations, + ) + temporary_patch = Path( + self.talos_patch_path_log.read_text(encoding="utf-8") + ) + self.assertFalse(temporary_patch.exists()) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + + def test_talos_failure_prevents_kubernetes_credential_mutation(self) -> None: + """Stop before Flux fan-out when any node cannot accept or use auth.""" + for operation in ("auth", "pull", "revision"): + with self.subTest(operation=operation): + result = self._run_helper( + self._valid_config(), + FAKE_TALOS_FAIL_NODE="10.0.0.2", + FAKE_TALOS_FAIL_OPERATION=operation, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + + def test_current_talos_nodes_skip_talos_api(self) -> None: + """Avoid a Talos availability dependency after this revision is proved.""" + result = self._run_helper( + self._valid_config(), + FAKE_TALOS_NODES_CURRENT="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(self.talos_log.exists()) + self.assertTrue(self.patch_capture.exists()) + + def test_dr_without_operator_uses_exact_declared_image(self) -> None: + """Verify stale bootstrap nodes before the operator Deployment exists.""" + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_KSAIL_OPERATOR_DEPLOYMENT_ABSENT="true", + FAKE_VARIABLES_BASE_ABSENT="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + pulls = [ + line + for line in self.talos_log.read_text(encoding="utf-8").splitlines() + if line.startswith("talos-pull:") + ] + self.assertEqual( + pulls, + [ + "talos-pull:10.0.0.2:ghcr.io/devantler-tech/ksail:v7.167.0", + "talos-pull:10.0.0.1:ghcr.io/devantler-tech/ksail:v7.167.0", + ], + ) + + def test_invalid_node_inventory_fails_closed(self) -> None: + """Reject empty, duplicate, and ambiguous Talos node InternalIPs.""" + invalid_inventories = [ + {"items": []}, + { + "items": [ + { + "metadata": {"name": "one"}, + "status": {"addresses": []}, + } + ] + }, + { + "items": [ + { + "metadata": {"name": "one"}, + "status": { + "addresses": [ + {"type": "InternalIP", "address": "10.0.0.1"}, + {"type": "InternalIP", "address": "10.0.0.2"}, + ] + }, + } + ] + }, + { + "items": [ + { + "metadata": {"name": "one"}, + "status": {"addresses": [ + {"type": "InternalIP", "address": "10.0.0.1"} + ]}, + }, + { + "metadata": {"name": "two"}, + "status": {"addresses": [ + {"type": "InternalIP", "address": "10.0.0.1"} + ]}, + }, + ] + }, + ] + for inventory in invalid_inventories: + with self.subTest(inventory=inventory): + result = self._run_helper( + self._valid_config(), + FAKE_NODE_JSON=json.dumps(inventory), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.talos_log.exists()) + self.assertFalse(self.patch_capture.exists()) + + def test_node_discovery_failure_prevents_kubernetes_credential_mutation(self) -> None: + """Fail closed before mutation when production nodes cannot be listed.""" + result = self._run_helper( + self._valid_config(), + FAKE_NODE_DISCOVERY_FAIL="true", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.talos_log.exists()) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.patch_capture.exists()) + + def test_ksail_lifecycle_wrapper_uses_only_sops_pull_token(self) -> None: + """Run create, reconcile, and update with the decrypted pull token.""" + self.assertTrue(KSAIL_PULL_WRAPPER.is_file()) + for command in ( + ("cluster", "create"), + ("workload", "reconcile"), + ("cluster", "update"), + ): + with self.subTest(command=command): + result = self._run_ksail_pull_wrapper(self._valid_config(), command) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + self.ksail_token_capture.read_text(encoding="utf-8"), + "fixture-secret-token", + ) + self.assertEqual( + self.ksail_username_capture.read_text(encoding="utf-8"), + "devantler", + ) + self.assertEqual( + self.ksail_command_capture.read_text(encoding="utf-8").strip(), + f"--config ksail.prod.yaml {' '.join(command)}", + ) + self.assertRegex( + self.ksail_revision_capture.read_text(encoding="utf-8"), + r"^[0-9a-f]{64}$", + ) + self.assertNotIn( + "fixture-secret-token", + result.stdout + result.stderr, + ) + temporary_config = Path( + self.output_path_log.read_text(encoding="utf-8") + ) + self.assertFalse(temporary_config.exists()) + + def test_ksail_publish_wrapper_preserves_actions_write_token(self) -> None: + """Inject only the SOPS revision while preserving publication auth.""" + result = self._run_ksail_pull_wrapper( + self._valid_config(), + ("workload", "push"), + GITHUB_ACTOR="fixture-publisher", + GHCR_TOKEN="fixture-actions-write-token", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + self.ksail_token_capture.read_text(encoding="utf-8"), + "fixture-actions-write-token", + ) + self.assertEqual( + self.ksail_username_capture.read_text(encoding="utf-8"), + "fixture-publisher", + ) + self.assertRegex( + self.ksail_revision_capture.read_text(encoding="utf-8"), + r"^[0-9a-f]{64}$", + ) + self.assertNotIn( + "fixture-actions-write-token", + result.stdout + result.stderr, + ) + self.assertFalse( + self.output_path_log.exists(), + "publication needs the ciphertext revision but not decryption", + ) + + def test_lifecycle_preserves_username_from_sops_docker_config(self) -> None: + """Avoid hard-coding the pull credential's registry username.""" + config = self._valid_config() + config["auths"]["ghcr.io"]["username"] = "pull-robot" # type: ignore[index] + + result = self._run_ksail_pull_wrapper( + config, + ("cluster", "update"), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + self.ksail_username_capture.read_text(encoding="utf-8"), + "pull-robot", + ) + + def test_ciphertext_rotation_changes_revision_without_hashing_token(self) -> None: + """Drive template drift from SOPS ciphertext rather than plaintext auth.""" + first = self._run_ksail_pull_wrapper( + self._valid_config(), + ("cluster", "update"), + ) + self.assertEqual(first.returncode, 0, first.stderr) + first_revision = self.ksail_revision_capture.read_text(encoding="utf-8") + + self._write_encrypted_secret("ENC[AES256_GCM,data:fixture-two]") + second = self._run_ksail_pull_wrapper( + self._valid_config(), + ("cluster", "update"), + ) + self.assertEqual(second.returncode, 0, second.stderr) + second_revision = self.ksail_revision_capture.read_text(encoding="utf-8") + + normalized_plaintext = json.dumps( + self._valid_config(), sort_keys=True, separators=(",", ":") + ) + plaintext_hash = hashlib.sha256( + f"{normalized_plaintext}\n".encode() + ).hexdigest() + self.assertNotEqual(first_revision, second_revision) + self.assertNotEqual(first_revision, plaintext_hash) + self.assertNotEqual(second_revision, plaintext_hash) + + def test_wrapper_rejects_arbitrary_commands(self) -> None: + """Never pass the decrypted pull credential to an arbitrary process.""" + result = self._run_ksail_pull_wrapper( + self._valid_config(), + ("workload", "delete"), + ) + + self.assertEqual(result.returncode, 64) + self.assertFalse(self.ksail_token_capture.exists()) + + def test_plaintext_revision_source_fails_closed(self) -> None: + """Refuse to hash a source value that is not SOPS ciphertext.""" + self._write_encrypted_secret("accidentally-plaintext") + + result = self._run_ksail_pull_wrapper( + self._valid_config(), + ("cluster", "update"), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.ksail_token_capture.exists()) + def test_accepts_standard_auth_only_docker_config(self) -> None: """Accept Docker configs that store only the standard encoded auth field.""" auth = base64.b64encode(b"devantler:fixture-secret-token").decode() @@ -580,19 +1149,92 @@ def test_materialised_consumer_mismatch_is_not_hidden(self) -> None: class DeployActionOrderingTests(unittest.TestCase): """Keep credential refreshes on both sides of KSail-owned mutations.""" + def test_cluster_lifecycle_uses_sops_auth_but_publish_keeps_actions_token( + self, + ) -> None: + """Separate KSail pull lifecycle auth from the Actions publish token.""" + action = ACTION.read_text(encoding="utf-8") + workflow = DR_REBUILD.read_text(encoding="utf-8") + wrapper = "./scripts/run-ksail-prod-with-pull-auth.sh" + + action_reconcile = action.index("id: reconcile") + action_update = action.index("name: 🔄 Update cluster") + action_reassert = action.index("id: reassert_flux_ghcr_auth") + self.assertIn( + f"run: {wrapper} workload reconcile", + action[action_reconcile:action_update], + ) + self.assertNotIn("GHCR_TOKEN:", action[action_reconcile:action_update]) + self.assertIn( + f"run: {wrapper} cluster update", + action[action_update:action_reassert], + ) + self.assertNotIn("GHCR_TOKEN:", action[action_update:action_reassert]) + + action_push = action.index("name: 📦 Push manifests to GHCR") + action_sign = action.index("name: ⚙️ Install cosign") + self.assertIn( + f"run: {wrapper} workload push", + action[action_push:action_sign], + ) + self.assertIn( + "GHCR_TOKEN: ${{ inputs.ghcr-token }}", + action[action_push:action_sign], + ) + + dr_create = workflow.index("name: 🏗️ Create cluster") + dr_stage = workflow.index("id: stage_flux_ghcr_auth") + self.assertIn( + f"run: {wrapper} cluster create", + workflow[dr_create:dr_stage], + ) + self.assertNotIn("GHCR_TOKEN:", workflow[dr_create:dr_stage]) + + dr_push = workflow.index("name: 📦 Push manifests to GHCR") + dr_verify = workflow.index("id: verify_flux_ghcr_auth_after_push") + self.assertIn( + f"run: {wrapper} workload push", + workflow[dr_push:dr_verify], + ) + self.assertIn( + "GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}", + workflow[dr_push:dr_verify], + ) + + dr_reconcile = workflow.index("name: 🔁 Trigger Flux reconciliation") + dr_wait = workflow.index("name: ⏳ Wait for Flux to settle") + self.assertIn( + f"run: {wrapper} workload reconcile", + workflow[dr_reconcile:dr_wait], + ) + self.assertNotIn("GHCR_TOKEN:", workflow[dr_reconcile:dr_wait]) + + def test_talosctl_is_installed_before_any_mutating_bridge(self) -> None: + """Ensure both isolated Actions jobs can execute node synchronization.""" + action = ACTION.read_text(encoding="utf-8") + workflow = DR_REBUILD.read_text(encoding="utf-8") + + for document in (action, workflow): + with self.subTest(document=document[:40]): + setup = document.index("name: ⚙️ Setup talosctl") + stage = document.index("id: stage_flux_ghcr_auth") + self.assertLess(setup, stage) + setup_step = document[setup:stage] + self.assertIn("TALOS_VERSION: \"1.13.5\"", setup_step) + self.assertIn("talosctl-linux-amd64", setup_step) + def test_consumer_staging_precedes_publish_and_is_reasserted_after_update( self, ) -> None: """Keep full credential staging before publish and after cluster update.""" action = ACTION.read_text(encoding="utf-8") + wrapper = "./scripts/run-ksail-prod-with-pull-auth.sh" first_refresh = action.index("id: stage_flux_ghcr_auth") - push = action.index("run: ksail --config ksail.prod.yaml workload push") + push = action.index(f"run: {wrapper} workload push") post_push_refresh = action.index("id: verify_flux_ghcr_auth_after_push") reconcile = action.index("id: reconcile") - cluster_update = action.index( - "run: ksail --config ksail.prod.yaml cluster update" - ) + cluster_update = action.index(f"run: {wrapper} cluster update") final_refresh = action.index("id: reassert_flux_ghcr_auth\n") self.assertLess(first_refresh, push) @@ -629,15 +1271,14 @@ def test_disaster_rebuild_preflights_then_stages_before_publish( ) -> None: """Keep DR credential checks around publish and OpenBao restoration.""" workflow = DR_REBUILD.read_text(encoding="utf-8") + wrapper = "./scripts/run-ksail-prod-with-pull-auth.sh" preflight = workflow.index( "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only" ) - cluster_create = workflow.index( - "run: ksail --config ksail.prod.yaml cluster create" - ) + cluster_create = workflow.index(f"run: {wrapper} cluster create") stage = workflow.index("id: stage_flux_ghcr_auth") - push = workflow.index("run: ksail --config ksail.prod.yaml workload push") + push = workflow.index(f"run: {wrapper} workload push") verify = workflow.index("id: verify_flux_ghcr_auth_after_push") fanout_verify = workflow.index("id: verify_flux_ghcr_fanout") openbao_restore = workflow.index( @@ -646,9 +1287,7 @@ def test_disaster_rebuild_preflights_then_stages_before_publish( post_restore_verify = workflow.index( "id: reassert_flux_ghcr_after_restore" ) - reconcile = workflow.index( - "run: ksail --config ksail.prod.yaml workload reconcile" - ) + reconcile = workflow.index(f"run: {wrapper} workload reconcile") self.assertLess(preflight, cluster_create) self.assertLess(cluster_create, stage) @@ -694,6 +1333,51 @@ def test_provider_upjet_unifi_reference_is_preflighted(self) -> None: self.assertIn(f'"{package_reference}"', helper) + def test_exact_declared_ksail_operator_image_is_preflighted(self) -> None: + """Bind the GHCR preflight to the chart's exact KSail image version.""" + release = KSAIL_OPERATOR_HELM_RELEASE.read_text(encoding="utf-8") + helper = HELPER.read_text(encoding="utf-8") + + version = next( + line.split(":", 1)[1].strip() + for line in release.splitlines() + if line.strip().startswith("version:") + ) + + self.assertIn( + '"devantler-tech/ksail:v${KSAIL_OPERATOR_VERSION}"', helper + ) + self.assertIn(".spec.chart.spec.version", helper) + self.assertEqual(version, "7.167.0") + + +class TalosRegistryAuthConfigTests(unittest.TestCase): + """Keep lifecycle drift and post-pull verification states distinct.""" + + def test_static_desired_revision_cannot_claim_verified_pull(self) -> None: + """Require helper proof before a new Talos node is considered current.""" + static_revision = TALOS_GHCR_REVISION.read_text(encoding="utf-8") + static_config = "\n".join( + line + for line in static_revision.splitlines() + if not line.lstrip().startswith("#") + ) + helper = HELPER.read_text(encoding="utf-8") + + self.assertIn("ghcr-pull-desired-revision", static_config) + self.assertNotIn("ghcr-pull-verified-revision", static_config) + self.assertIn("ghcr-pull-verified-revision", helper) + + def test_registry_auth_uses_supported_talos_document(self) -> None: + """Use one standalone RegistryAuthConfig document with variable auth.""" + registry_auth = TALOS_GHCR_AUTH.read_text(encoding="utf-8") + + self.assertIn("kind: RegistryAuthConfig", registry_auth) + self.assertIn("username: ${GHCR_USERNAME}", registry_auth) + self.assertIn("password: ${GHCR_TOKEN}", registry_auth) + self.assertNotIn("machine:\n", registry_auth) + self.assertNotIn("\n---\n", registry_auth) + if __name__ == "__main__": unittest.main() diff --git a/talos/cluster/authenticate-ghcr-pulls.yaml b/talos/cluster/authenticate-ghcr-pulls.yaml index da40e8f0f..887b9e820 100644 --- a/talos/cluster/authenticate-ghcr-pulls.yaml +++ b/talos/cluster/authenticate-ghcr-pulls.yaml @@ -3,50 +3,42 @@ # talos/cluster/verify-first-party-images.yaml (ImageVerificationConfig) verifies the # cosign signature of every ghcr.io/devantler-tech/* image at the containerd # PULL layer. That signature is a separate artifact stored in the SAME package -# as the image, and Talos fetches it using THIS machine.registries config — +# as the image, and Talos fetches it using THIS RegistryAuthConfig — # NOT the per-pod `ghcr-auth` imagePullSecret (which kubelet only hands to # containerd for the image-layer pull). Our first-party packages are PRIVATE, -# so without a node-level ghcr.io credential the signature fetch goes out -# anonymous → HTTP 401 → verification fails → ImagePullBackOff. +# so without working node-level ghcr.io credentials the signature fetch is +# denied → verification fails → ImagePullBackOff. # # It's latent: an image already cached on a node is "already present" and never # re-verified, so it only bites a FRESH pull — a new tenant release, or an # existing image after a node roll evicts its cache (how ascoachingogvaner # v1.2.0 wedged on prod-worker-1 after the k8s-1.36 roll). # -# This is pure host AUTH (machine.registries.config), NOT a mirror -# (machine.registries.mirrors): it attaches a credential to ghcr.io pulls with +# This is pure host auth, not a mirror: it attaches a credential to ghcr.io pulls with # no endpoint redirect. (ksail's --mirror-registry flag is the wrong tool here — # it rewrites the endpoint to a local pull-through container that doesn't exist # on Hetzner, and it's only ever processed at `cluster create`, never on # `cluster update`.) It is the same credential the cluster already holds for the # Flux manifests artifact (spec.cluster.localRegistry), so it adds no new trust. # -# ${GHCR_TOKEN} is expanded by ksail when it loads this patch (envvar.ExpandBytes -# in LoadPatches) from the deploy step's env — the SAME env var localRegistry -# already requires — so nothing secret is committed. This is ksail's documented, -# supported config-as-code pattern for node registry auth (declarative-config -# docs, "Environment Variable Expansion"; ksail#5135) — explicitly NOT the -# --mirror-registry CLI flag. Prod-only by construction: this dir pairs with -# ksail.prod.yaml; local/CI uses talos-local/, which carries no -# ImageVerificationConfig. +# ${GHCR_TOKEN} is expanded by ksail when it loads this patch. Production +# cluster lifecycle commands inject the validated Git/SOPS pull token through +# run-ksail-prod-with-pull-auth.sh; workload push preserves the separate Actions +# write token. The separate mark-ghcr-pull-revision.yaml patch carries a +# non-secret SOPS ciphertext revision because KSail deliberately redacts +# registry credentials from config fingerprints; the marker makes token-only +# rotations update the cluster-autoscaler template used by future nodes. +# Nothing secret is committed. Prod-only by construction: local/CI uses +# talos-local/. # -# CAVEAT — applying to EXISTING nodes: ksail's `cluster update` diff only tracks -# node count / Talos version / ISO, so a machine-config-only change like this is -# NOT pushed to already-running nodes (it reports "No changes detected"). This -# patch makes every node correct-by-construction on create/recreate (and the -# autoscaler's rendered worker config); to apply it to nodes that are already up, -# patch them once out-of-band, e.g.: -# talosctl --nodes patch mc --patch @ -# (registries is a no-reboot field), then restart any pod stuck in -# ImagePullBackOff. See docs/dr/runbook.md. +# Existing nodes are synchronized separately by refresh-flux-ghcr-auth.sh using +# this same RegistryAuthConfig shape, one node at a time in no-reboot mode, and +# each node must then pull the exact live ksail-operator image successfully. # # Reference: -# https://docs.siderolabs.com/talos/v1.13/reference/configuration/network/registriesconfig/ -machine: - registries: - config: - ghcr.io: - auth: - username: devantler - password: ${GHCR_TOKEN} +# https://docs.siderolabs.com/talos/v1.13/reference/configuration/cri/registryauthconfig +apiVersion: v1alpha1 +kind: RegistryAuthConfig +name: ghcr.io +username: ${GHCR_USERNAME} +password: ${GHCR_TOKEN} diff --git a/talos/cluster/mark-ghcr-pull-revision.yaml b/talos/cluster/mark-ghcr-pull-revision.yaml new file mode 100644 index 000000000..49f24bbd8 --- /dev/null +++ b/talos/cluster/mark-ghcr-pull-revision.yaml @@ -0,0 +1,14 @@ +# Non-secret rotation marker for the node-level GHCR pull credential. +# +# KSail redacts RegistryAuthConfig credentials when it fingerprints Talos +# machine configuration and autoscaler templates. Hashing the committed SOPS +# ciphertext gives token rotations a visible, non-secret input without +# publishing a stable hash of the decrypted token. Production lifecycle +# commands provide ${GHCR_PULL_REVISION}. This desired marker intentionally +# differs from the helper-owned ghcr-pull-verified-revision annotation: a newly +# created/autoscaled node must not claim a successful pull before it has proved +# one. The helper records that second marker only after an exact private KSail +# image pull succeeds. +machine: + nodeAnnotations: + platform.devantler.tech/ghcr-pull-desired-revision: ${GHCR_PULL_REVISION} From 33ed9a88643d78f4ac0e40c3ca670fe3c70c3286 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 02:52:43 +0200 Subject: [PATCH 12/16] fix(deploy): keep partial GHCR fan-out atomic --- docs/dr/runbook.md | 6 ++-- scripts/refresh-flux-ghcr-auth.sh | 27 ++++++++------ scripts/tests/test_refresh_flux_ghcr_auth.py | 38 ++++++++++++++++++++ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index 9a24b46fe..2e8c7bdda 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -184,8 +184,10 @@ talosctl version --client # must be installed; use the prod-pinned v1.13.5 ./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile # 5. Wait for Flux to settle -flux get kustomizations -A -# Re-run if any are NotReady; expect convergence in 10-15 minutes +for k in bootstrap infrastructure-controllers infrastructure apps; do + kubectl --context admin@prod -n flux-system wait "kustomization/${k}" \ + --for=condition=Ready --timeout=20m +done ./scripts/refresh-flux-ghcr-auth.sh # prove completed fan-out + every stale node # 6. ONLY if the OpenBao raft-snapshot recovery was impossible (no snapshot diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index c8e8a3b7a..8f41d3afa 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -411,6 +411,15 @@ patch_root_secret() { --patch-file="${patch_file}" } +patch_variables_base() { + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + patch secret variables-base \ + --type=merge \ + --patch-file="${variables_patch_file}" +} + # A fresh DR cluster does not have variables-base or the ESO fan-out resources # until its first Flux reconcile. In that case the current artifact creates the # chain from the same SOPS value, so only the root bootstrap patch is needed. @@ -433,19 +442,13 @@ if [[ -z "${variables_base_name}" ]]; then exit 0 fi -# Existing clusters must update and verify the whole SOPS -> variables-base -> -# PushSecret -> OpenBao -> ExternalSecret chain before switching root Flux auth. -# Otherwise a failed fan-out can still let normal root-source polling apply the -# newly-published artifact while tenant OCI/image pulls keep stale credentials. +# Prepare the variables-base payload locally, but do not mutate its live Secret +# until normal mode has proved the complete fan-out exists. Otherwise a failed +# normal deploy could leave PushSecret free to propagate an unmerged credential +# even though root Flux auth stayed unchanged. jq '{data: {ghcr_dockerconfigjson: .data[".dockerconfigjson"]}}' \ "${patch_file}" \ > "${variables_patch_file}" -kubectl \ - --context "${KUBE_CONTEXT}" \ - --namespace flux-system \ - patch secret variables-base \ - --type=merge \ - --patch-file="${variables_patch_file}" # A partially-bootstrapped DR cluster can already have variables-base while ESO # CRDs or individual fan-out objects do not exist yet. That state still needs @@ -501,11 +504,15 @@ if [[ "${fanout_complete}" != "true" ]]; then echo "::error::The GHCR fan-out is incomplete; root Flux auth was not changed. Use --allow-incomplete-fanout only during the DR bootstrap, then run the full verifier after reconciliation." exit 1 fi + patch_variables_base patch_root_secret echo "✅ Staged the Git/SOPS credential and refreshed root Flux auth; the first reconcile will complete the missing downstream fan-out." exit 0 fi +# Existing clusters update and verify the whole SOPS -> variables-base -> +# PushSecret -> OpenBao -> ExternalSecret chain before switching root Flux auth. +patch_variables_base force_sync_resource pushsecret flux-system seed-ghcr for namespace in "${FANOUT_NAMESPACES[@]}"; do force_sync_resource externalsecret "${namespace}" ghcr-auth diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index 8be306bf1..cf47da38e 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -18,6 +18,7 @@ KSAIL_PULL_WRAPPER = ROOT / "scripts" / "run-ksail-prod-with-pull-auth.sh" ACTION = ROOT / ".github" / "actions" / "deploy-prod" / "action.yml" DR_REBUILD = ROOT / ".github" / "workflows" / "dr-rebuild.yaml" +DR_RUNBOOK = ROOT / "docs" / "dr" / "runbook.md" KSAIL_OPERATOR_HELM_RELEASE = ( ROOT / "k8s" @@ -1086,9 +1087,21 @@ def test_partial_fanout_fails_closed_without_bootstrap_mode(self) -> None: ) self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.variables_patch_capture.exists()) self.assertFalse(self.patch_capture.exists()) self.assertFalse(self.fanout_log.exists()) + def test_missing_eso_crds_fails_without_staging_variables(self) -> None: + """Leave the fan-out source unchanged when normal mode is incomplete.""" + result = self._run_helper( + self._valid_config(), + FAKE_FANOUT_CRDS_ABSENT="true", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.patch_capture.exists()) + def test_partial_bootstrap_without_eso_crds_repairs_root(self) -> None: """Permit DR root repair before External Secrets CRDs exist.""" result = self._run_helper( @@ -1315,6 +1328,31 @@ def test_disaster_rebuild_preflights_then_stages_before_publish( ) self.assertEqual(workflow.count("scripts/refresh-flux-ghcr-auth.sh"), 5) + def test_manual_dr_waits_for_flux_before_full_bridge(self) -> None: + """Keep bootstrap mode until every first-reconcile layer is Ready.""" + runbook = DR_RUNBOOK.read_text(encoding="utf-8") + manual = runbook[ + runbook.index("# 2. Prove the Git/SOPS pull credential") : + runbook.index("# 6. ONLY if the OpenBao raft-snapshot") + ] + + bootstrap = manual.index( + "./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout" + ) + reconcile = manual.index( + "./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile" + ) + wait = manual.index( + "kubectl --context admin@prod -n flux-system wait" + ) + full_bridge = manual.index( + "./scripts/refresh-flux-ghcr-auth.sh # prove completed fan-out" + ) + + self.assertLess(bootstrap, reconcile) + self.assertLess(reconcile, wait) + self.assertLess(wait, full_bridge) + class RequiredPackageCoverageTests(unittest.TestCase): """Keep pinned private provider references in the live GHCR preflight.""" From 7d5c33866595131977b30c6d3d00e863bd27986c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 03:26:49 +0200 Subject: [PATCH 13/16] fix(deploy): harden GHCR recovery ordering --- .github/workflows/ci.yaml | 1 + .github/workflows/dr-rebuild.yaml | 2 +- docs/dr/runbook.md | 4 +-- scripts/refresh-flux-ghcr-auth.sh | 1 + scripts/tests/test_refresh_flux_ghcr_auth.py | 26 +++++++++++++++++++- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f573f4ea0..c1ad7c0c1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,6 +45,7 @@ jobs: - 'scripts/ghcr-auth-lib.sh' - 'scripts/run-ksail-prod-with-pull-auth.sh' - 'scripts/tests/test_refresh_flux_ghcr_auth.py' + - 'docs/dr/runbook.md' - '.github/actions/deploy-prod/**' - '.github/workflows/dr-rebuild.yaml' - '.github/workflows/ci.yaml' diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index db34855f9..47df07f6c 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -410,7 +410,7 @@ jobs: - name: 🔑 Reassert GHCR credential after OpenBao restore id: reassert_flux_ghcr_after_restore - if: ${{ inputs.restore }} + if: ${{ !cancelled() && inputs.restore && steps.verify_flux_ghcr_fanout.outcome == 'success' }} # The raft snapshot may contain an older infrastructure/ghcr/auth value. # Re-stage Git/SOPS in PushSecret-first order, wait for every consumer, # compare the materialized payloads, and only then reassert root auth. diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index 2e8c7bdda..3bbd4b3bb 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -53,8 +53,8 @@ these simultaneously and you cannot recover. > `ksail --config ksail.prod.yaml` create/update/push/reconcile command. The > wrapper injects the authoritative SOPS pull credential and its rotation > marker. Export `HCLOUD_TOKEN`, `WG_SERVER_PRIVATE_KEY`, and a usable SOPS Age -> key for create/update; publication additionally needs `GHCR_TOKEN` and -> `GITHUB_ACTOR`. +> key for create/update; workload reconciliation also requires the SOPS key. +> Publication additionally needs `GHCR_TOKEN` and `GITHUB_ACTOR`. --- diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index 8f41d3afa..d4e60d7ac 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -504,6 +504,7 @@ if [[ "${fanout_complete}" != "true" ]]; then echo "::error::The GHCR fan-out is incomplete; root Flux auth was not changed. Use --allow-incomplete-fanout only during the DR bootstrap, then run the full verifier after reconciliation." exit 1 fi + patch_root_secret patch_variables_base patch_root_secret echo "✅ Staged the Git/SOPS credential and refreshed root Flux auth; the first reconcile will complete the missing downstream fan-out." diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index cf47da38e..7de23b40c 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -18,6 +18,7 @@ KSAIL_PULL_WRAPPER = ROOT / "scripts" / "run-ksail-prod-with-pull-auth.sh" ACTION = ROOT / ".github" / "actions" / "deploy-prod" / "action.yml" DR_REBUILD = ROOT / ".github" / "workflows" / "dr-rebuild.yaml" +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yaml" DR_RUNBOOK = ROOT / "docs" / "dr" / "runbook.md" KSAIL_OPERATOR_HELM_RELEASE = ( ROOT @@ -1078,6 +1079,22 @@ def test_partial_bootstrap_repairs_root_without_forcing_missing_fanout( self.assertTrue(self.patch_capture.exists()) self.assertFalse(self.fanout_log.exists()) self.assertIn("first reconcile will complete", result.stdout) + self.assertEqual( + self.operation_log.read_text(encoding="utf-8").splitlines()[-3:], + ["root-patch", "variables-patch", "root-patch"], + ) + + def test_partial_bootstrap_repairs_root_before_staging_variables(self) -> None: + """Keep an unavailable root patch from advancing partial consumers.""" + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_MISSING_FANOUT_RESOURCE="pushsecret/flux-system/seed-ghcr", + FAKE_KUBECTL_FAIL="true", + ) + + self.assertEqual(result.returncode, 43) + self.assertFalse(self.variables_patch_capture.exists()) def test_partial_fanout_fails_closed_without_bootstrap_mode(self) -> None: """Reject a partial fan-out during a normal production deployment.""" @@ -1323,7 +1340,8 @@ def test_disaster_rebuild_preflights_then_stages_before_publish( workflow[fanout_verify:], ) self.assertIn( - "if: ${{ inputs.restore }}", + "if: ${{ !cancelled() && inputs.restore && " + "steps.verify_flux_ghcr_fanout.outcome == 'success' }}", workflow[post_restore_verify:], ) self.assertEqual(workflow.count("scripts/refresh-flux-ghcr-auth.sh"), 5) @@ -1331,6 +1349,7 @@ def test_disaster_rebuild_preflights_then_stages_before_publish( def test_manual_dr_waits_for_flux_before_full_bridge(self) -> None: """Keep bootstrap mode until every first-reconcile layer is Ready.""" runbook = DR_RUNBOOK.read_text(encoding="utf-8") + ci_workflow = CI_WORKFLOW.read_text(encoding="utf-8") manual = runbook[ runbook.index("# 2. Prove the Git/SOPS pull credential") : runbook.index("# 6. ONLY if the OpenBao raft-snapshot") @@ -1352,6 +1371,11 @@ def test_manual_dr_waits_for_flux_before_full_bridge(self) -> None: self.assertLess(bootstrap, reconcile) self.assertLess(reconcile, wait) self.assertLess(wait, full_bridge) + self.assertIn( + "workload reconciliation also requires the SOPS key", + runbook, + ) + self.assertIn("- 'docs/dr/runbook.md'", ci_workflow) class RequiredPackageCoverageTests(unittest.TestCase): From bd609251add2363659efac5f40b26c436a05764d Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 04:03:32 +0200 Subject: [PATCH 14/16] fix(deploy): scope production pull auth override --- .github/workflows/dr-rebuild.yaml | 2 + ksail.prod.yaml | 2 +- scripts/refresh-flux-ghcr-auth.sh | 3 + scripts/run-ksail-prod-with-pull-auth.sh | 7 +- scripts/tests/test_refresh_flux_ghcr_auth.py | 98 ++++++++++++++++++-- 5 files changed, 104 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index 47df07f6c..f6ac708fe 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -412,6 +412,8 @@ jobs: id: reassert_flux_ghcr_after_restore if: ${{ !cancelled() && inputs.restore && steps.verify_flux_ghcr_fanout.outcome == 'success' }} # The raft snapshot may contain an older infrastructure/ghcr/auth value. + # !cancelled() is intentional: even a failed restore can already have + # rematerialised that stale value, so attempt the fail-closed repair. # Re-stage Git/SOPS in PushSecret-first order, wait for every consumer, # compare the materialized payloads, and only then reassert root auth. run: ./scripts/refresh-flux-ghcr-auth.sh diff --git a/ksail.prod.yaml b/ksail.prod.yaml index 04d7466e6..fb8373168 100644 --- a/ksail.prod.yaml +++ b/ksail.prod.yaml @@ -137,7 +137,7 @@ spec: - siderolabs/util-linux-tools - siderolabs/qemu-guest-agent localRegistry: - registry: "${GHCR_USERNAME}:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests" + registry: "devantler:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests" # Cosign keyless signature verification on the flux-system OCIRepository that # KSail generates and owns (rendered onto spec.verify; a hand-written override # would lose the reconcile fight, so it is configured here per ksail#4987). diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index d4e60d7ac..be53a5dd2 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -37,6 +37,9 @@ readonly SYNC_INTERVAL="${FLUX_GHCR_SYNC_INTERVAL:-2}" KSAIL_OPERATOR_VERSION="$(yq -er '.spec.chart.spec.version' \ k8s/bases/infrastructure/controllers/ksail-operator/helm-release.yaml)" readonly KSAIL_OPERATOR_VERSION +# Both tenant release workflows create/update latest alongside every semver +# artifact and image tag. Flux still selects the signed semver artifact; latest +# is the stable read-permission/existence probe for the same private packages. readonly -a REQUIRED_PULL_TARGETS=( "devantler-tech/platform/manifests:latest" "devantler-tech/wedding-app/manifests:latest" diff --git a/scripts/run-ksail-prod-with-pull-auth.sh b/scripts/run-ksail-prod-with-pull-auth.sh index 2a558f535..722f8e375 100755 --- a/scripts/run-ksail-prod-with-pull-auth.sh +++ b/scripts/run-ksail-prod-with-pull-auth.sh @@ -22,6 +22,10 @@ readonly SCRIPT_DIR source "${SCRIPT_DIR}/ghcr-auth-lib.sh" readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" +# Viper applies this nested config override before KSail expands the two child +# environment variables. The template itself contains no credential. +# shellcheck disable=SC2016 +readonly PULL_REGISTRY_TEMPLATE='${GHCR_USERNAME}:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests' pull_revision="$(flux_ghcr_revision "${SECRET_FILE}")" readonly pull_revision @@ -52,7 +56,8 @@ else write_flux_ghcr_credentials "${docker_config}" "${credentials_file}" pull_username="$(jq -er '.username' "${credentials_file}")" pull_token="$(jq -er '.password' "${credentials_file}")" - GHCR_USERNAME="${pull_username}" \ + KSAIL_SPEC_CLUSTER_LOCALREGISTRY_REGISTRY="${PULL_REGISTRY_TEMPLATE}" \ + GHCR_USERNAME="${pull_username}" \ GHCR_TOKEN="${pull_token}" \ GHCR_PULL_REVISION="${pull_revision}" \ ksail --config ksail.prod.yaml "$1" "$2" diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index 7de23b40c..3f935410c 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -16,6 +16,7 @@ ROOT = Path(__file__).resolve().parents[2] HELPER = ROOT / "scripts" / "refresh-flux-ghcr-auth.sh" KSAIL_PULL_WRAPPER = ROOT / "scripts" / "run-ksail-prod-with-pull-auth.sh" +KSAIL_PROD_CONFIG = ROOT / "ksail.prod.yaml" ACTION = ROOT / ".github" / "actions" / "deploy-prod" / "action.yml" DR_REBUILD = ROOT / ".github" / "workflows" / "dr-rebuild.yaml" CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yaml" @@ -69,6 +70,11 @@ def setUp(self) -> None: self.ksail_username_capture = self.workspace / "ksail-username" self.ksail_revision_capture = self.workspace / "ksail-revision" self.ksail_command_capture = self.workspace / "ksail-command" + self.ksail_config_path_capture = self.workspace / "ksail-config-path" + self.ksail_registry_capture = self.workspace / "ksail-registry" + self.ksail_registry_override_capture = ( + self.workspace / "ksail-registry-override" + ) self.sync_state_dir = self.workspace / "sync-state" self.sync_state_dir.mkdir() self._write_encrypted_secret("ENC[AES256_GCM,data:fixture-one]") @@ -98,10 +104,30 @@ def setUp(self) -> None: exit 0 fi - if [[ "$arguments" == *" --config ksail.prod.yaml cluster create "* \ - || "$arguments" == *" --config ksail.prod.yaml cluster update "* \ - || "$arguments" == *" --config ksail.prod.yaml workload push "* \ - || "$arguments" == *" --config ksail.prod.yaml workload reconcile "* ]]; then + if [[ "$arguments" == *" cluster create "* \ + || "$arguments" == *" cluster update "* \ + || "$arguments" == *" workload push "* \ + || "$arguments" == *" workload reconcile "* ]]; then + config="" + previous="" + for argument in "$@"; do + if [[ "$previous" == --config ]]; then + config="$argument" + break + fi + previous="$argument" + done + test -f "$config" + registry="$(yq -er '.spec.cluster.localRegistry.registry' "$config")" + test "$registry" = \ + 'devantler:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests' + registry_override="${KSAIL_SPEC_CLUSTER_LOCALREGISTRY_REGISTRY:-}" + if [[ "$arguments" == *" workload push "* ]]; then + test -z "$registry_override" + else + test "$registry_override" = \ + '${GHCR_USERNAME}:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests' + fi test -n "${GHCR_TOKEN:-}" test -n "${GHCR_USERNAME:-}" test "${GHCR_PULL_REVISION:-}" != "" @@ -109,6 +135,9 @@ def setUp(self) -> None: printf '%s' "$GHCR_USERNAME" > "$KSAIL_USERNAME_CAPTURE" printf '%s' "$GHCR_PULL_REVISION" > "$KSAIL_REVISION_CAPTURE" printf '%s\n' "$*" > "$KSAIL_COMMAND_CAPTURE" + printf '%s' "$config" > "$KSAIL_CONFIG_PATH_CAPTURE" + printf '%s' "$registry" > "$KSAIL_REGISTRY_CAPTURE" + printf '%s' "$registry_override" > "$KSAIL_REGISTRY_OVERRIDE_CAPTURE" exit 0 fi @@ -532,6 +561,9 @@ def _run_helper( self.ksail_username_capture, self.ksail_revision_capture, self.ksail_command_capture, + self.ksail_config_path_capture, + self.ksail_registry_capture, + self.ksail_registry_override_capture, ): marker.unlink(missing_ok=True) for marker in self.sync_state_dir.iterdir(): @@ -556,6 +588,11 @@ def _run_helper( "KSAIL_USERNAME_CAPTURE": str(self.ksail_username_capture), "KSAIL_REVISION_CAPTURE": str(self.ksail_revision_capture), "KSAIL_COMMAND_CAPTURE": str(self.ksail_command_capture), + "KSAIL_CONFIG_PATH_CAPTURE": str(self.ksail_config_path_capture), + "KSAIL_REGISTRY_CAPTURE": str(self.ksail_registry_capture), + "KSAIL_REGISTRY_OVERRIDE_CAPTURE": str( + self.ksail_registry_override_capture + ), "EXPECTED_PULL_USERNAME": expected_username, "EXPECTED_PULL_TOKEN": expected_token, "EXPECTED_GHCR_REVISION": self._expected_revision(), @@ -588,6 +625,9 @@ def _run_ksail_pull_wrapper( self.ksail_username_capture, self.ksail_revision_capture, self.ksail_command_capture, + self.ksail_config_path_capture, + self.ksail_registry_capture, + self.ksail_registry_override_capture, ): marker.unlink(missing_ok=True) expected_username, expected_token = self._expected_credentials(config) @@ -602,6 +642,11 @@ def _run_ksail_pull_wrapper( "KSAIL_USERNAME_CAPTURE": str(self.ksail_username_capture), "KSAIL_REVISION_CAPTURE": str(self.ksail_revision_capture), "KSAIL_COMMAND_CAPTURE": str(self.ksail_command_capture), + "KSAIL_CONFIG_PATH_CAPTURE": str(self.ksail_config_path_capture), + "KSAIL_REGISTRY_CAPTURE": str(self.ksail_registry_capture), + "KSAIL_REGISTRY_OVERRIDE_CAPTURE": str( + self.ksail_registry_override_capture + ), "EXPECTED_PULL_USERNAME": expected_username, "EXPECTED_PULL_TOKEN": expected_token, "EXPECTED_GHCR_REVISION": self._expected_revision(), @@ -833,8 +878,26 @@ def test_ksail_lifecycle_wrapper_uses_only_sops_pull_token(self) -> None: "devantler", ) self.assertEqual( - self.ksail_command_capture.read_text(encoding="utf-8").strip(), - f"--config ksail.prod.yaml {' '.join(command)}", + self.ksail_command_capture.read_text(encoding="utf-8") + .strip() + .split(), + ["--config", "ksail.prod.yaml", *command], + ) + self.assertEqual( + self.ksail_config_path_capture.read_text(encoding="utf-8"), + "ksail.prod.yaml", + ) + self.assertEqual( + self.ksail_registry_capture.read_text(encoding="utf-8"), + "devantler:${GHCR_TOKEN}" + "@ghcr.io/devantler-tech/platform/manifests", + ) + self.assertEqual( + self.ksail_registry_override_capture.read_text( + encoding="utf-8" + ), + "${GHCR_USERNAME}:${GHCR_TOKEN}" + "@ghcr.io/devantler-tech/platform/manifests", ) self.assertRegex( self.ksail_revision_capture.read_text(encoding="utf-8"), @@ -867,6 +930,18 @@ def test_ksail_publish_wrapper_preserves_actions_write_token(self) -> None: self.ksail_username_capture.read_text(encoding="utf-8"), "fixture-publisher", ) + self.assertEqual( + self.ksail_config_path_capture.read_text(encoding="utf-8"), + "ksail.prod.yaml", + ) + self.assertEqual( + self.ksail_registry_capture.read_text(encoding="utf-8"), + "devantler:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests", + ) + self.assertEqual( + self.ksail_registry_override_capture.read_text(encoding="utf-8"), + "", + ) self.assertRegex( self.ksail_revision_capture.read_text(encoding="utf-8"), r"^[0-9a-f]{64}$", @@ -880,6 +955,17 @@ def test_ksail_publish_wrapper_preserves_actions_write_token(self) -> None: "publication needs the ciphertext revision but not decryption", ) + def test_production_config_keeps_its_protected_registry_template(self) -> None: + """Keep dynamic pull auth in the wrapper-owned environment only.""" + config = KSAIL_PROD_CONFIG.read_text(encoding="utf-8") + + self.assertIn( + 'registry: "devantler:${GHCR_TOKEN}' + '@ghcr.io/devantler-tech/platform/manifests"', + config, + ) + self.assertNotIn("${GHCR_USERNAME}:${GHCR_TOKEN}@ghcr.io", config) + def test_lifecycle_preserves_username_from_sops_docker_config(self) -> None: """Avoid hard-coding the pull credential's registry username.""" config = self._valid_config() From a1711f5078bb546f6d30c21f977ff28efe5f06f0 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 04:38:46 +0200 Subject: [PATCH 15/16] fix(ci): isolate bridge validation-only paths --- .github/workflows/ci.yaml | 13 +++- scripts/tests/test_refresh_flux_ghcr_auth.py | 72 +++++++++++++++++--- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c1ad7c0c1..e4dda0025 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,7 @@ jobs: pull-requests: read # detect changed files outputs: k8s: ${{ steps.filter.outputs.k8s }} + bridge_validation: ${{ steps.filter.outputs.bridge_validation }} talos: ${{ steps.filter.outputs.talos }} steps: - name: 📑 Checkout @@ -41,6 +42,13 @@ jobs: - 'talos/**' - 'scripts/validate-naming.py' - 'scripts/validate-embedded-json.py' + - 'scripts/refresh-flux-ghcr-auth.sh' + - 'scripts/ghcr-auth-lib.sh' + - 'scripts/run-ksail-prod-with-pull-auth.sh' + - '.github/actions/deploy-prod/**' + - '.github/workflows/dr-rebuild.yaml' + - '.github/workflows/ci.yaml' + bridge_validation: - 'scripts/refresh-flux-ghcr-auth.sh' - 'scripts/ghcr-auth-lib.sh' - 'scripts/run-ksail-prod-with-pull-auth.sh' @@ -62,7 +70,10 @@ jobs: # longer boots it; instead it schema-validates and security-scans the # manifests. This needs no secrets, so — unlike the old system test — it also # runs on fork PRs. - if: github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true' + if: >- + github.event_name == 'pull_request' && + (needs.changes.outputs.k8s == 'true' || + needs.changes.outputs.bridge_validation == 'true') runs-on: ubuntu-latest permissions: contents: read # checkout repository diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index 3f935410c..96a0e968f 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -30,6 +30,11 @@ / "ksail-operator" / "helm-release.yaml" ) +KSAIL_OPERATOR_VERSION = next( + line.split(":", 1)[1].strip() + for line in KSAIL_OPERATOR_HELM_RELEASE.read_text(encoding="utf-8").splitlines() + if line.strip().startswith("version:") +) PROVIDER_UPJET_UNIFI = ( ROOT / "k8s" @@ -701,7 +706,7 @@ def test_refreshes_root_and_fanout_without_leaking_plaintext(self) -> None: "devantler-tech/ascoachingogvaner/manifests:latest", "devantler-tech/wedding-app:latest", "devantler-tech/ascoachingogvaner:latest", - "devantler-tech/ksail:v7.167.0", + f"devantler-tech/ksail:v{KSAIL_OPERATOR_VERSION}", "devantler-tech/provider-upjet-unifi:v0.1.0", ], ) @@ -787,8 +792,10 @@ def test_dr_without_operator_uses_exact_declared_image(self) -> None: self.assertEqual( pulls, [ - "talos-pull:10.0.0.2:ghcr.io/devantler-tech/ksail:v7.167.0", - "talos-pull:10.0.0.1:ghcr.io/devantler-tech/ksail:v7.167.0", + "talos-pull:10.0.0.2:ghcr.io/devantler-tech/ksail:" + f"v{KSAIL_OPERATOR_VERSION}", + "talos-pull:10.0.0.1:ghcr.io/devantler-tech/ksail:" + f"v{KSAIL_OPERATOR_VERSION}", ], ) @@ -1463,6 +1470,56 @@ def test_manual_dr_waits_for_flux_before_full_bridge(self) -> None: ) self.assertIn("- 'docs/dr/runbook.md'", ci_workflow) + def test_bridge_docs_and_tests_validate_without_deploying(self) -> None: + """Keep validation-only bridge changes out of production deploys.""" + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + filters = workflow[ + workflow.index(" filters: |") : workflow.index("\n validate:") + ] + k8s_filter = filters[ + filters.index(" k8s:") : filters.index( + " bridge_validation:" + ) + ] + bridge_filter = filters[ + filters.index(" bridge_validation:") : filters.index( + " talos:" + ) + ] + + for validation_only_path in ( + "scripts/tests/test_refresh_flux_ghcr_auth.py", + "docs/dr/runbook.md", + ): + with self.subTest(path=validation_only_path): + quoted_path = f"- '{validation_only_path}'" + self.assertNotIn(quoted_path, k8s_filter) + self.assertIn(quoted_path, bridge_filter) + + self.assertIn( + "bridge_validation: ${{ steps.filter.outputs.bridge_validation }}", + workflow, + ) + validate = workflow[ + workflow.index(" validate:") : workflow.index(" naming:") + ] + self.assertIn( + "needs.changes.outputs.bridge_validation == 'true'", validate + ) + deploy = workflow[ + workflow.index(" deploy-prod:") : workflow.index( + " heal-prod-on-failure:" + ) + ] + heal = workflow[ + workflow.index(" heal-prod-on-failure:") : workflow.index( + " ci-required-checks:" + ) + ] + for production_job in (deploy, heal): + self.assertIn("needs.changes.outputs.k8s == 'true'", production_job) + self.assertNotIn("bridge_validation", production_job) + class RequiredPackageCoverageTests(unittest.TestCase): """Keep pinned private provider references in the live GHCR preflight.""" @@ -1483,20 +1540,13 @@ def test_provider_upjet_unifi_reference_is_preflighted(self) -> None: def test_exact_declared_ksail_operator_image_is_preflighted(self) -> None: """Bind the GHCR preflight to the chart's exact KSail image version.""" - release = KSAIL_OPERATOR_HELM_RELEASE.read_text(encoding="utf-8") helper = HELPER.read_text(encoding="utf-8") - version = next( - line.split(":", 1)[1].strip() - for line in release.splitlines() - if line.strip().startswith("version:") - ) - self.assertIn( '"devantler-tech/ksail:v${KSAIL_OPERATOR_VERSION}"', helper ) self.assertIn(".spec.chart.spec.version", helper) - self.assertEqual(version, "7.167.0") + self.assertTrue(KSAIL_OPERATOR_VERSION) class TalosRegistryAuthConfigTests(unittest.TestCase): From 2fa35762cd6fa730e12e324ad6e229615e9f0e23 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 14 Jul 2026 05:04:15 +0200 Subject: [PATCH 16/16] fix(deploy): enforce yq recovery prerequisite --- AGENTS.md | 7 ++- docs/dr/runbook.md | 4 +- scripts/ghcr-auth-lib.sh | 15 +++++ scripts/refresh-flux-ghcr-auth.sh | 5 ++ scripts/run-ksail-prod-with-pull-auth.sh | 1 + scripts/tests/test_refresh_flux_ghcr_auth.py | 62 +++++++++++++++++++- 6 files changed, 89 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fd07906f7..256053a36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,9 @@ sudo apt-get update && sudo apt-get install -y age wget -O /tmp/sops_amd64.deb https://github.com/getsops/sops/releases/download/v3.8.1/sops_3.8.1_amd64.deb sudo dpkg -i /tmp/sops_amd64.deb +# yq v4 — exact YAML field queries in production lifecycle/recovery scripts +brew install yq + # KSail — cluster + workload lifecycle (Homebrew) brew tap devantler-tech/formulas && brew install ksail ``` @@ -69,7 +72,7 @@ Verify the toolchain: ```bash docker --version && ksail --version && kubectl version --client -sops --version && age --version +sops --version && age --version && yq --version docker ps # Docker daemon is running ksail cluster list # existing Talos clusters ``` @@ -338,7 +341,7 @@ export GITHUB_ACTOR=devantler ### Tool Reinstallation -If tools stop working, reinstall in order: Docker (restart the service if needed) → KSail (`brew reinstall ksail`) → kubectl (check the cluster context) → SOPS and Age (check the encryption keys). +If tools stop working, reinstall in order: Docker (restart the service if needed) → KSail (`brew reinstall ksail`) → kubectl (check the cluster context) → SOPS, Age, and yq v4 (check the encryption keys and `yq --version`). ## What's Useful for the AI Assistant diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index 3bbd4b3bb..addcb0a24 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -54,7 +54,9 @@ these simultaneously and you cannot recover. > wrapper injects the authoritative SOPS pull credential and its rotation > marker. Export `HCLOUD_TOKEN`, `WG_SERVER_PRIVATE_KEY`, and a usable SOPS Age > key for create/update; workload reconciliation also requires the SOPS key. -> Publication additionally needs `GHCR_TOKEN` and `GITHUB_ACTOR`. +> Install `yq v4` before invoking the wrapper or credential bridge; both fail +> fast before decryption or mutation when the compatible YAML query tool is +> unavailable. Publication additionally needs `GHCR_TOKEN` and `GITHUB_ACTOR`. --- diff --git a/scripts/ghcr-auth-lib.sh b/scripts/ghcr-auth-lib.sh index ecc3e47b5..18a61e9af 100644 --- a/scripts/ghcr-auth-lib.sh +++ b/scripts/ghcr-auth-lib.sh @@ -1,6 +1,21 @@ #!/usr/bin/env bash # Shared secret-safe helpers for the Git/SOPS GHCR pull credential. +# Fail before any decrypt or mutation when the YAML query implementation used +# for exact field selection is missing or incompatible. +require_flux_ghcr_yaml_tool() { + local yq_version + + if ! command -v yq >/dev/null 2>&1 \ + || ! yq_version="$(yq --version 2>/dev/null)" \ + || [[ "${yq_version}" != *" version v4."* ]] \ + || ! printf 'probe: ok\n' \ + | yq -er '.probe | select(tag == "!!str")' - >/dev/null 2>&1; then + echo "::error::Mike Farah yq v4 is required by the GHCR credential bridge; install it with 'brew install yq' before running production lifecycle or recovery commands." >&2 + return 127 + fi +} + # Decrypt only the Docker config scalar into a caller-owned restricted file. decrypt_flux_ghcr_docker_config() { local output_file="$1" diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index be53a5dd2..ab937d52f 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -12,6 +12,7 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR # shellcheck source=scripts/ghcr-auth-lib.sh source "${SCRIPT_DIR}/ghcr-auth-lib.sh" +require_flux_ghcr_yaml_tool check_only=false allow_incomplete_fanout=false @@ -182,6 +183,10 @@ sync_talos_registry_auth() { echo "::error::Could not list Talos nodes; refusing to mutate any Kubernetes credential consumers." return 1 fi + # talosctl connects to the public control-plane endpoints in talosconfig and + # proxies --nodes targets through them. Target addresses therefore must be + # the stable InternalIPs as seen by those endpoint servers, not client-facing + # ExternalIPs (Talos v1.13 "Endpoints and nodes"). if ! jq -e ' (.items | length) > 0 and all(.items[]; diff --git a/scripts/run-ksail-prod-with-pull-auth.sh b/scripts/run-ksail-prod-with-pull-auth.sh index 722f8e375..5840225d2 100755 --- a/scripts/run-ksail-prod-with-pull-auth.sh +++ b/scripts/run-ksail-prod-with-pull-auth.sh @@ -20,6 +20,7 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR # shellcheck source=scripts/ghcr-auth-lib.sh source "${SCRIPT_DIR}/ghcr-auth-lib.sh" +require_flux_ghcr_yaml_tool readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" # Viper applies this nested config override before KSail expands the two child diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py index 96a0e968f..3929c7829 100644 --- a/scripts/tests/test_refresh_flux_ghcr_auth.py +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -14,7 +14,9 @@ ROOT = Path(__file__).resolve().parents[2] +AGENT_INSTRUCTIONS = ROOT / "AGENTS.md" HELPER = ROOT / "scripts" / "refresh-flux-ghcr-auth.sh" +GHCR_AUTH_LIB = ROOT / "scripts" / "ghcr-auth-lib.sh" KSAIL_PULL_WRAPPER = ROOT / "scripts" / "run-ksail-prod-with-pull-auth.sh" KSAIL_PROD_CONFIG = ROOT / "ksail.prod.yaml" ACTION = ROOT / ".github" / "actions" / "deploy-prod" / "action.yml" @@ -355,7 +357,8 @@ def setUp(self) -> None: } }, status: {addresses: [ - {type: "InternalIP", address: "10.0.0.2"} + {type: "InternalIP", address: "10.0.0.2"}, + {type: "ExternalIP", address: "198.51.100.2"} ]} }, { @@ -370,7 +373,8 @@ def setUp(self) -> None: } }, status: {addresses: [ - {type: "InternalIP", address: "10.0.0.1"} + {type: "InternalIP", address: "10.0.0.1"}, + {type: "ExternalIP", address: "198.51.100.1"} ]} } ] @@ -1345,6 +1349,9 @@ def test_talosctl_is_installed_before_any_mutating_bridge(self) -> None: setup_step = document[setup:stage] self.assertIn("TALOS_VERSION: \"1.13.5\"", setup_step) self.assertIn("talosctl-linux-amd64", setup_step) + if document == action: + restore = document.index("name: 🔑 Restore talosconfig") + self.assertLess(restore, stage) def test_consumer_staging_precedes_publish_and_is_reasserted_after_update( self, @@ -1521,6 +1528,57 @@ def test_bridge_docs_and_tests_validate_without_deploying(self) -> None: self.assertNotIn("bridge_validation", production_job) +class ManualBridgePrerequisiteTests(unittest.TestCase): + """Keep manual recovery dependencies explicit and fail-fast.""" + + def test_yq_v4_is_documented_and_preflighted(self) -> None: + """Require the YAML tool used before any GHCR recovery mutation.""" + instructions = AGENT_INSTRUCTIONS.read_text(encoding="utf-8") + runbook = DR_RUNBOOK.read_text(encoding="utf-8") + library = GHCR_AUTH_LIB.read_text(encoding="utf-8") + + self.assertIn("yq v4", instructions) + self.assertIn("yq v4", runbook[: runbook.index("## Scenario 1")]) + self.assertIn("require_flux_ghcr_yaml_tool()", library) + for entrypoint in (HELPER, KSAIL_PULL_WRAPPER): + with self.subTest(entrypoint=entrypoint.name): + self.assertIn( + "require_flux_ghcr_yaml_tool", + entrypoint.read_text(encoding="utf-8"), + ) + + def test_yaml_tool_preflight_rejects_missing_or_incompatible_yq(self) -> None: + """Stop before entrypoint work when Mike Farah yq v4 is unavailable.""" + cases = ("missing", "incompatible") + for case in cases: + with self.subTest(case=case), tempfile.TemporaryDirectory() as temp_dir: + if case == "incompatible": + fake_yq = Path(temp_dir) / "yq" + fake_yq.write_text( + "#!/bin/bash\necho 'yq 3.4.3'\n", encoding="utf-8" + ) + fake_yq.chmod(0o755) + environment = os.environ.copy() + environment["PATH"] = temp_dir + + result = subprocess.run( + [ + "/bin/bash", + "-c", + 'source "$1"; require_flux_ghcr_yaml_tool', + "bash", + str(GHCR_AUTH_LIB), + ], + env=environment, + text=True, + capture_output=True, + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("yq v4 is required", result.stderr) + + class RequiredPackageCoverageTests(unittest.TestCase): """Keep pinned private provider references in the live GHCR preflight."""