diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index a70bfd29e..14e79f9b6 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -19,7 +19,7 @@ inputs: description: talosconfig for the prod cluster (secrets.TALOS_CONFIG) used by `ksail cluster update`. required: true ghcr-token: - description: GHCR PAT with write:packages (secrets.GHCR_TOKEN) for OCI push/pull and cosign auth. + description: GHCR PAT with write:packages (secrets.GHCR_TOKEN) for OCI push and signing; every pull path uses the Git/SOPS credential. required: true hcloud-token: description: Hetzner Cloud API token (secrets.HCLOUD_TOKEN) used by the KSail Hetzner provider and CCM/CSI. @@ -50,6 +50,19 @@ runs: sudo install /tmp/ksail /usr/local/bin/ksail ksail --version + - name: ⚙️ Setup talosctl + # refresh-flux-ghcr-auth.sh uses the host API for a secret-safe, + # per-node RegistryAuthConfig patch and exact image-pull proof. + shell: bash + env: + # Keep in step with spec.cluster.talos.version in ksail.prod.yaml. + # renovate: datasource=github-releases depName=siderolabs/talos extractVersion=^v(?.+)$ + TALOS_VERSION: "1.13.5" + run: | + curl -fsSL "https://github.com/siderolabs/talos/releases/download/v${TALOS_VERSION}/talosctl-linux-amd64" -o /tmp/talosctl + sudo install /tmp/talosctl /usr/local/bin/talosctl + talosctl version --client + - name: 🔐 Create SOPS Age key shell: bash env: @@ -126,6 +139,15 @@ runs: echo "::error::Prod cluster (context admin@prod) is unreachable with the restored kubeconfig (last kubectl error: ${last_err}). KUBE_CONFIG (and likely TALOS_CONFIG) are probably stale after a cluster rebuild — refresh both and update the 'prod' environment secrets. See docs/dr/runbook.md → 'Refresh CI deploy credentials'." exit 1 + - name: 🔑 Stage Flux and consumer GHCR pull credential + id: stage_flux_ghcr_auth + # Validate all seven packages, synchronize and prove the exact live image + # pull on every stale Talos node, then stage and verify the Kubernetes fan-out + # BEFORE publishing latest. Every pull consumer has one known-readable + # credential before the mutable tag can be observed. + shell: bash + run: ./scripts/refresh-flux-ghcr-auth.sh + # Publish the GitOps manifests (push → sign → attest → reconcile) BEFORE # syncing Talos machine config (the "🔄 Update cluster" step runs LAST). # Manifest delivery never touches the nodes, so it must not be gated on @@ -139,7 +161,7 @@ runs: GITHUB_ACTOR: ${{ github.actor }} GHCR_TOKEN: ${{ inputs.ghcr-token }} HCLOUD_TOKEN: ${{ inputs.hcloud-token }} - run: ksail --config ksail.prod.yaml workload push + run: ./scripts/run-ksail-prod-with-pull-auth.sh workload push - name: ⚙️ Install cosign # sigstore/cosign-installer is the official installer action — it @@ -284,14 +306,20 @@ runs: push-to-registry: true create-storage-record: false + - name: 🔎 Verify Flux GHCR pull credential after publish + id: verify_flux_ghcr_auth_after_push + # Revalidate the newly-published artifact before apps reconcile. The + # complete consumer fan-out was staged before the mutable tag moved. + shell: bash + run: ./scripts/refresh-flux-ghcr-auth.sh --check-only + - name: 🔁 Trigger Flux reconciliation id: reconcile shell: bash env: GITHUB_ACTOR: ${{ github.actor }} - GHCR_TOKEN: ${{ inputs.ghcr-token }} HCLOUD_TOKEN: ${{ inputs.hcloud-token }} - run: ksail --config ksail.prod.yaml workload reconcile + run: ./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile - name: 🩺 Diagnose Flux on failure if: failure() && steps.reconcile.outcome == 'failure' @@ -314,10 +342,24 @@ runs: # release). shell: bash env: - GHCR_TOKEN: ${{ inputs.ghcr-token }} HCLOUD_TOKEN: ${{ inputs.hcloud-token }} # ksail env-expands ${WG_SERVER_PRIVATE_KEY} in the talos patches at # load time; an unset var renders an empty key and Talos rejects the # whole machine config ('private key is invalid: wrong key "" length: 0'). WG_SERVER_PRIVATE_KEY: ${{ inputs.wg-server-private-key }} - run: ksail --config ksail.prod.yaml cluster update + run: ./scripts/run-ksail-prod-with-pull-auth.sh cluster update + + - name: 🔑 Reassert Flux and consumer GHCR pull credential + id: reassert_flux_ghcr_auth + # cluster update is already wrapped with Git/SOPS auth, but reassert the + # full transaction afterward in case KSail fails part-way. Reassert after + # its synchronous consumer fan-out when that update fails part-way, but + # only after the publish, signing, attestation, post-publish auth check, + # and Flux reconcile all succeeded. Never mutate auth as cleanup for a + # failed delivery. + if: >- + always() && + steps.verify_flux_ghcr_auth_after_push.outcome == 'success' && + steps.reconcile.outcome == 'success' + shell: bash + run: ./scripts/refresh-flux-ghcr-auth.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 749ac42dd..e4dda0025 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,7 @@ jobs: pull-requests: read # detect changed files outputs: k8s: ${{ steps.filter.outputs.k8s }} + bridge_validation: ${{ steps.filter.outputs.bridge_validation }} talos: ${{ steps.filter.outputs.talos }} steps: - name: 📑 Checkout @@ -41,6 +42,20 @@ jobs: - 'talos/**' - 'scripts/validate-naming.py' - 'scripts/validate-embedded-json.py' + - 'scripts/refresh-flux-ghcr-auth.sh' + - 'scripts/ghcr-auth-lib.sh' + - 'scripts/run-ksail-prod-with-pull-auth.sh' + - '.github/actions/deploy-prod/**' + - '.github/workflows/dr-rebuild.yaml' + - '.github/workflows/ci.yaml' + bridge_validation: + - 'scripts/refresh-flux-ghcr-auth.sh' + - 'scripts/ghcr-auth-lib.sh' + - 'scripts/run-ksail-prod-with-pull-auth.sh' + - 'scripts/tests/test_refresh_flux_ghcr_auth.py' + - 'docs/dr/runbook.md' + - '.github/actions/deploy-prod/**' + - '.github/workflows/dr-rebuild.yaml' - '.github/workflows/ci.yaml' talos: - 'talos/**' @@ -55,7 +70,10 @@ jobs: # longer boots it; instead it schema-validates and security-scans the # manifests. This needs no secrets, so — unlike the old system test — it also # runs on fork PRs. - if: github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true' + if: >- + github.event_name == 'pull_request' && + (needs.changes.outputs.k8s == 'true' || + needs.changes.outputs.bridge_validation == 'true') runs-on: ubuntu-latest permissions: contents: read # checkout repository @@ -65,6 +83,20 @@ jobs: with: persist-credentials: false + - name: 🔐 Validate Flux GHCR credential bridge + # Pure local tests with fake ksail/docker/kubectl commands: no SOPS key, + # registry credential, cluster access, or network is used. + run: | + bash -n \ + scripts/ghcr-auth-lib.sh \ + scripts/refresh-flux-ghcr-auth.sh \ + scripts/run-ksail-prod-with-pull-auth.sh + shellcheck \ + scripts/ghcr-auth-lib.sh \ + scripts/refresh-flux-ghcr-auth.sh \ + scripts/run-ksail-prod-with-pull-auth.sh + python3 -m unittest scripts.tests.test_refresh_flux_ghcr_auth + - name: 🧩 Validate embedded JSON blobs # ConfigMaps that embed whole JSON documents as block scalars (e.g. the # Headlamp Kubescape exceptions ConfigMap's exceptionPolicies) are opaque @@ -230,12 +262,16 @@ jobs: # Placeholder key material for env-expanded patch values (ksail # expands these from environment secrets at deploy time — see - # talos/control-planes/wireguard.yaml and talos/cluster/registry-auth.yaml). + # talos/control-planes/wireguard.yaml and + # talos/cluster/authenticate-ghcr-pulls.yaml and + # talos/cluster/mark-ghcr-pull-revision.yaml). # Real-format but throwaway, so validation exercises the expansion # path without any real secret on the PR event. WG_SERVER_PRIVATE_KEY="$(openssl rand -base64 32)" + GHCR_USERNAME="placeholder-ghcr-user" GHCR_TOKEN="placeholder-ghcr-token" - export WG_SERVER_PRIVATE_KEY GHCR_TOKEN + GHCR_PULL_REVISION="$(printf '%s\n' 'ENC[ci-placeholder-ciphertext]' | shasum -a 256 | awk '{print $1}')" + export WG_SERVER_PRIVATE_KEY GHCR_USERNAME GHCR_TOKEN GHCR_PULL_REVISION validate_overlay() { local dir="$1" mode="$2" diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index aa83f06dd..f6ac708fe 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -85,6 +85,17 @@ jobs: sudo install /tmp/ksail /usr/local/bin/ksail ksail --version + - name: ⚙️ Setup talosctl + # The GHCR bridge verifies RegistryAuthConfig on each stale node. + env: + # Keep in step with spec.cluster.talos.version in ksail.prod.yaml. + # renovate: datasource=github-releases depName=siderolabs/talos extractVersion=^v(?.+)$ + TALOS_VERSION: "1.13.5" + run: | + curl -fsSL "https://github.com/siderolabs/talos/releases/download/v${TALOS_VERSION}/talosctl-linux-amd64" -o /tmp/talosctl + sudo install /tmp/talosctl /usr/local/bin/talosctl + talosctl version --client + - name: 🔐 Create SOPS Age key env: SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }} @@ -94,33 +105,51 @@ jobs: echo "${SOPS_AGE_KEY}" > ~/.config/sops/age/keys.txt chmod 600 ~/.config/sops/age/keys.txt + - name: 🔎 Preflight Flux GHCR pull credential + # Validate every package served by the shared Git/SOPS credential + # before creating infrastructure or publishing a mutable latest tag. + run: ./scripts/refresh-flux-ghcr-auth.sh --check-only + - name: 🏗️ Create cluster # From-zero provisioning: Hetzner servers, Talos boot, CCM, CSI. Writes # a fresh kubeconfig (~/.kube/config, context admin@prod) and # talosconfig (~/.talos/config) on this runner — every later step uses # those, so this workflow does NOT depend on the (now stale) # KUBE_CONFIG / TALOS_CONFIG environment secrets. - run: ksail --config ksail.prod.yaml cluster create + run: ./scripts/run-ksail-prod-with-pull-auth.sh cluster create env: - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} # ksail env-expands ${WG_SERVER_PRIVATE_KEY} in the talos patches at # load time (talos/control-planes/wireguard.yaml); an unset var # renders an empty key and Talos rejects the whole machine config. WG_SERVER_PRIVATE_KEY: ${{ secrets.WG_SERVER_PRIVATE_KEY }} + - name: 🔑 Stage Flux and consumer GHCR pull credential + id: stage_flux_ghcr_auth + # cluster create already uses the Git/SOPS pull token. Reassert it on + # every current Talos node and make Git/SOPS authoritative before the + # mutable tag is published. A + # partial bootstrap may have variables-base but no ESO resources yet; + # the bridge safely repairs root auth so Flux can create them. + run: ./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout + - name: 📦 Push manifests to GHCR - run: ksail --config ksail.prod.yaml workload push + run: ./scripts/run-ksail-prod-with-pull-auth.sh workload push env: GITHUB_ACTOR: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + - name: 🔎 Verify Flux GHCR pull credential after publish + id: verify_flux_ghcr_auth_after_push + # Prove the newly-published artifact is readable before reconciliation; + # every available consumer was already staged before push. + run: ./scripts/refresh-flux-ghcr-auth.sh --check-only + - name: 🔁 Trigger Flux reconciliation - run: ksail --config ksail.prod.yaml workload reconcile + run: ./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile env: GITHUB_ACTOR: ${{ github.actor }} - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - name: ⏳ Wait for Flux to settle @@ -132,6 +161,13 @@ jobs: done echo "✅ All Flux Kustomizations Ready — fresh platform converged." + - name: 🔑 Verify completed GHCR credential fan-out + id: verify_flux_ghcr_fanout + # Bootstrap mode may repair only root auth when ESO is not installed + # yet. After Flux converges, require the complete PushSecret / + # ExternalSecret transaction and exact consumer payload verification. + run: ./scripts/refresh-flux-ghcr-auth.sh + - name: 💾 Velero resource restore (newest Completed backup) if: ${{ inputs.restore }} run: | @@ -372,6 +408,16 @@ jobs: echo "::endgroup::" echo "✅ OpenBao restored from the raft snapshot mirror." + - name: 🔑 Reassert GHCR credential after OpenBao restore + id: reassert_flux_ghcr_after_restore + if: ${{ !cancelled() && inputs.restore && steps.verify_flux_ghcr_fanout.outcome == 'success' }} + # The raft snapshot may contain an older infrastructure/ghcr/auth value. + # !cancelled() is intentional: even a failed restore can already have + # rematerialised that stale value, so attempt the fail-closed repair. + # Re-stage Git/SOPS in PushSecret-first order, wait for every consumer, + # compare the materialized payloads, and only then reassert root auth. + run: ./scripts/refresh-flux-ghcr-auth.sh + - name: 🔑 Refresh CI deploy credentials (KUBE_CONFIG / TALOS_CONFIG) env: DR_GH_ADMIN_TOKEN: ${{ secrets.DR_GH_ADMIN_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index 62cf1ee5c..256053a36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,9 @@ sudo apt-get update && sudo apt-get install -y age wget -O /tmp/sops_amd64.deb https://github.com/getsops/sops/releases/download/v3.8.1/sops_3.8.1_amd64.deb sudo dpkg -i /tmp/sops_amd64.deb +# yq v4 — exact YAML field queries in production lifecycle/recovery scripts +brew install yq + # KSail — cluster + workload lifecycle (Homebrew) brew tap devantler-tech/formulas && brew install ksail ``` @@ -69,7 +72,7 @@ Verify the toolchain: ```bash docker --version && ksail --version && kubectl version --client -sops --version && age --version +sops --version && age --version && yq --version docker ps # Docker daemon is running ksail cluster list # existing Talos clusters ``` @@ -143,9 +146,12 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o 1. Merging a PR through the merge queue runs the `deploy-prod` job in `ci.yaml` (the normal path). A direct push to `main` bypasses the queue, so deploy it manually by running the `CD` workflow (`cd.yaml`, `workflow_dispatch`). Both run the same `ksail` steps below. 2. The `deploy-prod` composite action (shared by both paths) uses `ksail --config ksail.prod.yaml` to target the committed prod config. 3. `ksail.prod.yaml` has `kustomizationFile: clusters/prod`, so KSail/Flux use `k8s/clusters/prod/kustomization.yaml` as the entry point — no root `k8s/kustomization.yaml` or file rewriting is needed. -4. `ksail --config ksail.prod.yaml cluster create` (first run) or `cluster update` (subsequent runs) provisions / reconciles the Hetzner servers, Talos, CCM, and CSI. -5. `ksail --config ksail.prod.yaml workload push` packages manifests and pushes them to GHCR. -6. `ksail --config ksail.prod.yaml workload reconcile` triggers Flux to sync from the OCI artifact. +4. `scripts/run-ksail-prod-with-pull-auth.sh cluster create|update` provisions / reconciles the Hetzner servers, Talos, CCM, and CSI with the Git/SOPS pull credential; the wrapper also passes a SOPS-ciphertext revision so token-only rotations refresh the Cluster Autoscaler machine template. +5. The bridge decrypts only the Git/SOPS pull credential and performs real OCI manifest reads for all seven private consumers (the Platform and tenant manifest artifacts, both tenant application images, and the KSail plus provider-upjet-unifi packages used by Kyverno verification). On nodes whose verified revision is stale, it applies Talos `RegistryAuthConfig` workers-first, proves the exact private KSail image pull, and only then records the verified revision. It then updates `variables-base`, force-syncs and verifies the PushSecret plus tenant/Kyverno ExternalSecrets, and finally reasserts root auth — all before a mutable `latest` tag is published. The DR workflow first runs `--check-only` before creating infrastructure, then uses explicit `--allow-incomplete-fanout` bootstrap mode after cluster creation and requires a full bridge pass after Flux converges. +6. `scripts/run-ksail-prod-with-pull-auth.sh workload push` packages manifests and pushes them with the separate Actions write token. +7. `scripts/refresh-flux-ghcr-auth.sh --check-only` revalidates the newly-published artifact without mutating the cluster. +8. `scripts/run-ksail-prod-with-pull-auth.sh workload reconcile` triggers Flux with Git/SOPS pull auth. +9. After `cluster update`, the full bridge reasserts every pull path in case a partial update or older managed state was applied. DR also runs it after an OpenBao raft restore because the snapshot may contain an older GHCR value. **Key differences from local:** @@ -157,20 +163,39 @@ Production uses **Talos + Hetzner** via KSail's native Hetzner provider. KSail o ### Dual-Provider Model - **Local / CI:** `ksail cluster create` → Talos + Docker provider → local OCI registry → `ksail workload push` / `reconcile`. -- **Production:** `ksail --config ksail.prod.yaml cluster create|update` → Talos + Hetzner provider → Hetzner CCM + CSI installed by KSail → `ksail --config ksail.prod.yaml workload push` to GHCR → `workload reconcile`. +- **Production:** `scripts/run-ksail-prod-with-pull-auth.sh cluster create|update` → Talos + Hetzner provider → Hetzner CCM + CSI installed by KSail → the same wrapper's `workload push` to GHCR → `workload reconcile`. ## CI/CD Pipelines - **`ci.yaml`** — runs on `pull_request` (static manifest validation + Kubescape scan, no cluster) and `merge_group` (deploys prod via the Hetzner provider). Concurrency is shared with `cd.yaml` so a manual deploy and a merge-queue deploy can never run against the prod cluster at the same time. - **`cd.yaml`** — runs on `workflow_dispatch` (manual). Deploys to the production Hetzner cluster using `ksail --config ksail.prod.yaml`. Covers direct pushes to `main`, which bypass the merge queue and so are not deployed by `ci.yaml`. -- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (push → cosign-sign → attest SBOM + SLSA provenance → Flux reconcile → Talos `cluster update`), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. +- **`.github/actions/deploy-prod`** — the composite action both deploy paths call (stage/verify all GHCR pull consumers → push → cosign-sign → attest SBOM + SLSA provenance → revalidate published artifact → Flux reconcile → Talos `cluster update` → final reassert), so the merge-queue and manual deploys can never drift. Secrets are passed as inputs because composite actions cannot read `secrets`. **Required GitHub Secrets:** -- `GHCR_TOKEN` — long-lived PAT (owner: `devantler`) with `write:packages` scope, used for GHCR push/pull authentication. +- `GHCR_TOKEN` — long-lived PAT (owner: `devantler`) with `write:packages` scope, used only for OCI push/signing. It is **not** a pull credential. - `SOPS_AGE_KEY` — Age private key for SOPS secret decryption. - `HCLOUD_TOKEN` — Hetzner Cloud API token (read/write), used by the KSail Hetzner provider and by the Hetzner CCM / CSI at runtime. +The authoritative **production pull** credential for Flux, tenants, Kyverno, +and Talos hosts is +`stringData.ghcr_dockerconfigjson` in +`k8s/bases/bootstrap/secret.enc.yaml`. The deploy bridge refreshes +`flux-system/ksail-registry-credentials` from that value before Flux must fetch +the artifact and reasserts it after `cluster update` in case KSail rewrites its +managed Secret. Before publish on existing clusters, the bridge updates `variables-base`, +force-syncs `seed-ghcr` into OpenBao, force-syncs the tenant/Kyverno +ExternalSecrets, and verifies their materialised `ghcr-auth` payloads before +switching root Flux auth. Only explicit DR bootstrap mode may repair root auth +after staging `variables-base` while the fan-out is incomplete; DR must run the +full verifier after Flux converges. A direct credential commit to `main` still +needs a manual `CD` workflow dispatch because direct pushes bypass the merge-queue +deploy. +The lifecycle wrapper injects the same username/token into KSail's local +registry and Talos patches. A non-secret hash of the committed SOPS ciphertext +is the desired machine-template revision; the bridge stores a separate verified +revision on each existing node only after an exact image pull succeeds. + **Required GitHub Variables:** none. ## Working with Secrets @@ -304,15 +329,19 @@ With the KSail Hetzner provider the cluster is cattle — rebuild it in place: ```bash export HCLOUD_TOKEN=... -ksail --config ksail.prod.yaml cluster update # scales / re-provisions missing nodes +export WG_SERVER_PRIVATE_KEY=... +export SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt +export GHCR_TOKEN=... # publication only +export GITHUB_ACTOR=devantler +./scripts/run-ksail-prod-with-pull-auth.sh cluster update # For a full rebuild from zero, see docs/dr/runbook.md scenario 4. -ksail --config ksail.prod.yaml workload push -ksail --config ksail.prod.yaml workload reconcile +./scripts/run-ksail-prod-with-pull-auth.sh workload push +./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile ``` ### Tool Reinstallation -If tools stop working, reinstall in order: Docker (restart the service if needed) → KSail (`brew reinstall ksail`) → kubectl (check the cluster context) → SOPS and Age (check the encryption keys). +If tools stop working, reinstall in order: Docker (restart the service if needed) → KSail (`brew reinstall ksail`) → kubectl (check the cluster context) → SOPS, Age, and yq v4 (check the encryption keys and `yq --version`). ## What's Useful for the AI Assistant diff --git a/docs/dr/crypto-custody.md b/docs/dr/crypto-custody.md index 4e9972dc5..4cd961664 100644 --- a/docs/dr/crypto-custody.md +++ b/docs/dr/crypto-custody.md @@ -120,7 +120,8 @@ posture is: 1. **Do not commit it to git.** It is `.gitignore`d via the default `~/.talos/config` location. 2. **Keep one copy on each operator workstation** that needs to manage - nodes. Replacing a workstation = re-run `ksail --config ksail.prod.yaml cluster update` + nodes. Replacing a workstation = re-run + `./scripts/run-ksail-prod-with-pull-auth.sh cluster update` (which regenerates the local `~/.talos/config` from the cluster's running PKI), or use `talosctl config merge ` against a backup file. Talos does not have SSH-equivalent — see DR Scenario 4 diff --git a/docs/dr/runbook.md b/docs/dr/runbook.md index eb28e6ec5..addcb0a24 100644 --- a/docs/dr/runbook.md +++ b/docs/dr/runbook.md @@ -48,6 +48,16 @@ these simultaneously and you cannot recover. > and updating the unifi tenant's `cluster_wg_peer_public_key` to the new > public half. +> **Manual production lifecycle prerequisite:** use +> `scripts/run-ksail-prod-with-pull-auth.sh`, never a bare +> `ksail --config ksail.prod.yaml` create/update/push/reconcile command. The +> wrapper injects the authoritative SOPS pull credential and its rotation +> marker. Export `HCLOUD_TOKEN`, `WG_SERVER_PRIVATE_KEY`, and a usable SOPS Age +> key for create/update; workload reconciliation also requires the SOPS key. +> Install `yq v4` before invoking the wrapper or credential bridge; both fail +> fast before decryption or mutation when the compatible YAML query tool is +> unavailable. Publication additionally needs `GHCR_TOKEN` and `GITHUB_ACTOR`. + --- ## Scenario 1 — Single node loss @@ -62,7 +72,7 @@ kubectl get pods -A --field-selector=status.phase!=Running kubectl get pdb -A # all should show ALLOWED-DISRUPTIONS=1 # Replace the failed node (re-runs Hetzner provisioning for missing members) -ksail --config ksail.prod.yaml cluster update +./scripts/run-ksail-prod-with-pull-auth.sh cluster update ``` If any workload is stuck in Pending because all replicas were on the dead @@ -101,7 +111,7 @@ kubectl get pdb -A kubectl get deploy -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}\t{.spec.strategy.rollingUpdate.maxUnavailable}{"\n"}{end}' # Apply the upgrade (in-place rolling Talos OS + Kubernetes upgrade) -ksail --config ksail.prod.yaml cluster update +./scripts/run-ksail-prod-with-pull-auth.sh cluster update ``` If anything reports `maxUnavailable` other than `0`, that workload was @@ -156,21 +166,33 @@ Flux reconciliation. ```bash # 1. Set credentials locally export HCLOUD_TOKEN= +export WG_SERVER_PRIVATE_KEY= export GHCR_TOKEN= +export GITHUB_ACTOR=devantler export SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt # points at the env's Age key +talosctl version --client # must be installed; use the prod-pinned v1.13.5 + +# 2. Prove the Git/SOPS pull credential can read every private package before +# creating infrastructure or publishing a mutable latest tag +./scripts/refresh-flux-ghcr-auth.sh --check-only -# 2. Boot a fresh cluster (ksail handles Talos boot, CCM, CSI, kubeconfig) -ksail --config ksail.prod.yaml cluster create +# 3. Boot a fresh cluster (ksail handles Talos boot, CCM, CSI, kubeconfig) +./scripts/run-ksail-prod-with-pull-auth.sh cluster create -# 3. Bootstrap Flux from this repo -ksail --config ksail.prod.yaml workload push # packages -> GHCR -ksail --config ksail.prod.yaml workload reconcile # Flux pulls and applies +# 4. Bootstrap Flux from this repo +./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout +./scripts/run-ksail-prod-with-pull-auth.sh workload push +./scripts/refresh-flux-ghcr-auth.sh --check-only +./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile -# 4. Wait for Flux to settle -flux get kustomizations -A -# Re-run if any are NotReady; expect convergence in 10-15 minutes +# 5. Wait for Flux to settle +for k in bootstrap infrastructure-controllers infrastructure apps; do + kubectl --context admin@prod -n flux-system wait "kustomization/${k}" \ + --for=condition=Ready --timeout=20m +done +./scripts/refresh-flux-ghcr-auth.sh # prove completed fan-out + every stale node -# 4b. ONLY if the OpenBao raft-snapshot recovery was impossible (no snapshot +# 6. ONLY if the OpenBao raft-snapshot recovery was impossible (no snapshot # in R2 — the vault came up fresh): re-feed the user-fed secrets that # SOPS deliberately does not seed (see the push-secret-seed-* files in # k8s/bases/infrastructure/vault-seed/). Until then, @@ -407,7 +429,7 @@ kubectl -n kube-system get cm cluster-autoscaler-status -o yaml Common causes: - Pool `maxSize` reached — increase `max` under the relevant pool in `ksail.prod.yaml`, then run - `ksail --config ksail.prod.yaml cluster update` + `./scripts/run-ksail-prod-with-pull-auth.sh cluster update` - `HCLOUD_TOKEN` expired — rotate in SOPS secrets and GitHub environment secrets ### Orphaned autoscaler nodes after cluster delete @@ -430,7 +452,7 @@ hcloud server list # If the server exists but node doesn't appear in kubectl: # The worker machine config may be invalid or stale. # Re-run cluster update to regenerate worker config and re-apply: -ksail --config ksail.prod.yaml cluster update +./scripts/run-ksail-prod-with-pull-auth.sh cluster update ``` --- @@ -485,6 +507,83 @@ gh secret set TALOS_CONFIG --env prod --repo devantler-tech/platform < ~/.talos/ > step in the deploy workflows is the local guard against that behaviour until > those land. +## Scenario 10 — production GHCR pull credential rotation or denial + +The authoritative Flux, tenant, Kyverno, and Talos-host pull credential is the +SOPS-encrypted +`stringData.ghcr_dockerconfigjson` value in +`k8s/bases/bootstrap/secret.enc.yaml`. It is deliberately separate from the +GitHub `prod` environment's `GHCR_TOKEN`, which only pushes/signs artifacts. +Production KSail lifecycle commands run through +`scripts/run-ksail-prod-with-pull-auth.sh`, which derives username/token from +Git/SOPS and supplies a non-secret SOPS-ciphertext revision to the Talos +machine/autoscaler template. + +The production deploy closes the bootstrap loop in this order: + +1. Decrypt only `ghcr_dockerconfigjson` and perform real OCI manifest reads for + all seven private consumers: Platform manifests, both tenant manifest + artifacts, both tenant application images, and the KSail plus + provider-upjet-unifi packages used by Kyverno verification. During DR, a + read-only `--check-only` pass happens before infrastructure creation. +2. Before changing mutable `latest`, list every Kubernetes Node (including + NotReady/autoscaled nodes). For each node whose **verified** ciphertext + revision is stale, apply the supported Talos `RegistryAuthConfig` in + no-reboot mode, workers before control planes; pull the exact live KSail + operator image through the Talos CRI; and only then record the verified + revision. A distinct desired revision in the committed Talos configuration + refreshes future Cluster Autoscaler templates but never counts as proof. +3. Patch `variables-base`; force-sync + `seed-ghcr` into OpenBao; force-sync the tenant/Kyverno ExternalSecrets; + verify every materialised `ghcr-auth` payload matches Git/SOPS; and only then + patch `flux-system/ksail-registry-credentials`. A fresh or partial DR + bootstrap may have no ESO CRDs/resources yet; explicit + `--allow-incomplete-fanout` mode stages `variables-base` and repairs root auth + so the first reconcile can create the chain. Normal mode fails closed on any + missing fan-out resource. +4. Push and sign the artifact with `GHCR_TOKEN`, revalidate the newly-published + artifact with `--check-only`, and only then explicitly reconcile Flux. DR + runs the full bridge again after every Flux Kustomization is Ready, proving + that bootstrap mode completed the entire fan-out, and once more after an + OpenBao raft restore so a snapshot cannot rematerialise an older credential. +5. Re-run the bridge after `cluster update`; it repairs any node left stale by + a partial lifecycle operation and re-verifies the root plus downstream + fan-out. Nodes already carrying the verified revision skip Talos API calls, + so an ordinary deploy does not depend on every Talos endpoint. + +For a normal rotation, update the encrypted value through a PR and let the merge +queue deploy it. If the encrypted file was pushed directly to `main`, manually +dispatch the `CD` workflow: direct pushes do not run the production deploy. + +For an already-denied root source, run the same secret-safe bridge from a clean +`main` checkout with the production Age key and `admin@prod` kube context, then +reconcile the source: + +```bash +./scripts/refresh-flux-ghcr-auth.sh +flux reconcile source oci flux-system -n flux-system --context admin@prod +flux reconcile kustomization flux-system -n flux-system --context admin@prod +``` + +If tenant sources still show `DENIED` after an older/manual recovery path, force +the existing fan-out after `variables-base` has converged: + +```bash +stamp=$(date +%s) +kubectl --context admin@prod -n flux-system annotate pushsecret seed-ghcr \ + force-sync="$stamp" --overwrite +for namespace in wedding-app ascoachingogvaner kyverno; do + kubectl --context admin@prod -n "$namespace" annotate externalsecret ghcr-auth \ + force-sync="$stamp" --overwrite +done +flux get sources oci -A --context admin@prod +``` + +Do not replace this bridge with only a root ExternalSecret: Flux needs valid root +auth to fetch the artifact that installs/updates External Secrets, while OpenBao +receives this rotated value only through a downstream PushSecret. That circular +dependency cannot recover a stale root credential by itself. + --- ## Related documents diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml index e02123e4c..df1ec292e 100644 --- a/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml +++ b/k8s/bases/infrastructure/vault-seed/push-secret-seed-ghcr.yaml @@ -9,15 +9,17 @@ # (infra-ghcr-readonly) — a platform secret, not a tenant secret. Source key # `ghcr_dockerconfigjson` lives in secret.enc.yaml (shared, # cluster-independent — the same org token both clusters use) and holds the full -# `{"auths":{"ghcr.io":{...}}}` document. Re-pushed hourly, so a rotated -# token (or a re-initialized vault) converges without manual re-apply. +# `{"auths":{"ghcr.io":{...}}}` document. The production deploy bridge +# force-syncs this path before apps reconcile; the hourly interval is the +# fallback for out-of-band changes or a re-initialized vault. # # This one is deliberately SOPS-seeded, NOT user-fed, even though it is an # upstream token: the verify-image-signatures ClusterPolicy needs the kyverno # `ghcr-auth` Secret to fetch signature manifests of PRIVATE first-party -# images (ksail-operator runs in EVERY cluster, including ephemeral CI ones -# where no operator exists to feed the vault) — without it Kyverno denies -# those pods at admission. That makes it bootstrap machinery. +# images. KSail and provider-upjet packages run in managed clusters, including +# ephemeral CI ones where no operator exists to feed the vault; without this +# Secret, Kyverno denies those pods at admission. That makes it bootstrap +# machinery. apiVersion: external-secrets.io/v1alpha1 kind: PushSecret metadata: diff --git a/scripts/ghcr-auth-lib.sh b/scripts/ghcr-auth-lib.sh new file mode 100644 index 000000000..18a61e9af --- /dev/null +++ b/scripts/ghcr-auth-lib.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Shared secret-safe helpers for the Git/SOPS GHCR pull credential. + +# Fail before any decrypt or mutation when the YAML query implementation used +# for exact field selection is missing or incompatible. +require_flux_ghcr_yaml_tool() { + local yq_version + + if ! command -v yq >/dev/null 2>&1 \ + || ! yq_version="$(yq --version 2>/dev/null)" \ + || [[ "${yq_version}" != *" version v4."* ]] \ + || ! printf 'probe: ok\n' \ + | yq -er '.probe | select(tag == "!!str")' - >/dev/null 2>&1; then + echo "::error::Mike Farah yq v4 is required by the GHCR credential bridge; install it with 'brew install yq' before running production lifecycle or recovery commands." >&2 + return 127 + fi +} + +# Decrypt only the Docker config scalar into a caller-owned restricted file. +decrypt_flux_ghcr_docker_config() { + local output_file="$1" + local secret_file="${2:-k8s/bases/bootstrap/secret.enc.yaml}" + + ksail workload cipher decrypt \ + "${secret_file}" \ + --extract '["stringData"]["ghcr_dockerconfigjson"]' \ + --output "${output_file}" \ + >/dev/null + chmod 600 "${output_file}" +} + +# Validate the Docker config and write canonical username/password JSON. +write_flux_ghcr_credentials() { + local docker_config="$1" + local output_file="$2" + + if ! jq -e ' + def non_empty_string: type == "string" and length > 0; + (.auths["ghcr.io"] // {}) as $auth + | ((($auth | has("username")) or ($auth | has("password")))) + as $explicit_present + | (($auth.username | non_empty_string) + and ($auth.password | non_empty_string)) as $explicit_valid + | ($auth | has("auth")) as $encoded_present + | (if $encoded_present then + try ( + $auth.auth + | @base64d + | capture("^(?[^:]+):(?.+)$") + ) catch null + else null end) as $decoded + | (($decoded != null) + and ($decoded.username | non_empty_string) + and ($decoded.password | non_empty_string)) as $encoded_valid + | select( + ((($explicit_present | not) or $explicit_valid) + and (($encoded_present | not) or $encoded_valid) + and ($explicit_valid or $encoded_valid) + and (((($explicit_present and $encoded_present) | not)) + or (($auth.username == $decoded.username) + and ($auth.password == $decoded.password)))) + ) + | if $explicit_valid then + {username: $auth.username, password: $auth.password} + else + {username: $decoded.username, password: $decoded.password} + end + ' "${docker_config}" > "${output_file}"; then + echo "::error::The SOPS GHCR pull credential is not a valid Docker config with non-empty, consistent ghcr.io username/password and auth fields." + return 1 + fi + chmod 600 "${output_file}" +} + +# Print a non-secret ciphertext revision for redaction-resistant drift detection. +flux_ghcr_revision() { + local secret_file="${1:-k8s/bases/bootstrap/secret.enc.yaml}" + + # Hash the committed SOPS ciphertext, not the decrypted credential. This + # changes when the pull credential is rotated without publishing a stable + # verifier for the token itself in the Kubernetes Node annotation. + yq -er ' + .stringData.ghcr_dockerconfigjson + | select(tag == "!!str" and length > 0 and test("^ENC\\[")) + ' "${secret_file}" \ + | shasum -a 256 \ + | awk '{print $1}' +} diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh new file mode 100755 index 000000000..ab937d52f --- /dev/null +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -0,0 +1,533 @@ +#!/usr/bin/env bash +# Refresh the KSail-managed root Flux pull Secret from the Git/SOPS source. +# +# Flux cannot fetch the artifact containing a rotated credential while its +# bootstrap Secret is stale. Keep this bridge outside Flux so a deployment can +# repair that bootstrap edge before asking Flux to reconcile. + +set -euo pipefail +set +x + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +# shellcheck source=scripts/ghcr-auth-lib.sh +source "${SCRIPT_DIR}/ghcr-auth-lib.sh" +require_flux_ghcr_yaml_tool + +check_only=false +allow_incomplete_fanout=false +if (($# > 1)); then + echo "Usage: $0 [--check-only|--allow-incomplete-fanout]" >&2 + exit 64 +fi +if (($# == 1)); then + case "$1" in + --check-only) check_only=true ;; + --allow-incomplete-fanout) allow_incomplete_fanout=true ;; + *) + echo "Usage: $0 [--check-only|--allow-incomplete-fanout]" >&2 + exit 64 + ;; + esac +fi + +readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" +readonly KUBE_CONTEXT="${KUBE_CONTEXT:-admin@prod}" +readonly SYNC_ATTEMPTS="${FLUX_GHCR_SYNC_ATTEMPTS:-60}" +readonly SYNC_INTERVAL="${FLUX_GHCR_SYNC_INTERVAL:-2}" +KSAIL_OPERATOR_VERSION="$(yq -er '.spec.chart.spec.version' \ + k8s/bases/infrastructure/controllers/ksail-operator/helm-release.yaml)" +readonly KSAIL_OPERATOR_VERSION +# Both tenant release workflows create/update latest alongside every semver +# artifact and image tag. Flux still selects the signed semver artifact; latest +# is the stable read-permission/existence probe for the same private packages. +readonly -a REQUIRED_PULL_TARGETS=( + "devantler-tech/platform/manifests:latest" + "devantler-tech/wedding-app/manifests:latest" + "devantler-tech/ascoachingogvaner/manifests:latest" + "devantler-tech/wedding-app:latest" + "devantler-tech/ascoachingogvaner:latest" + "devantler-tech/ksail:v${KSAIL_OPERATOR_VERSION}" + "devantler-tech/provider-upjet-unifi:v0.1.0" +) +readonly -a FANOUT_NAMESPACES=( + "wedding-app" + "ascoachingogvaner" + "kyverno" +) + +if ! [[ "${SYNC_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]] \ + || ! [[ "${SYNC_INTERVAL}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + echo "::error::FLUX_GHCR_SYNC_ATTEMPTS and FLUX_GHCR_SYNC_INTERVAL must be non-negative numbers, with at least one attempt." + exit 64 +fi + +work_dir="$(mktemp -d)" +trap 'rm -rf "${work_dir}"' EXIT +chmod 700 "${work_dir}" +umask 077 + +docker_config="${work_dir}/config.json" +credentials_file="${work_dir}/credentials.json" +basic_curl_config="${work_dir}/curl-basic.config" +bearer_curl_config="${work_dir}/curl-bearer.config" +token_response="${work_dir}/token.json" +patch_file="${work_dir}/patch.json" +variables_patch_file="${work_dir}/variables-patch.json" +expected_normalized="${work_dir}/expected-normalized.json" +fanout_api_resources="${work_dir}/fanout-api-resources.txt" +talos_auth_patch_file="${work_dir}/talos-registry-auth.json" +talos_revision_patch_file="${work_dir}/talos-registry-revision.json" +talos_result_file="${work_dir}/talos-result.txt" +talos_nodes_file="${work_dir}/talos-nodes.json" +talos_node_targets="${work_dir}/talos-node-targets.tsv" +ksail_operator_deployment="${work_dir}/ksail-operator-deployment.json" + +# Force an ESO resource to reconcile and observe a post-annotation Ready edge. +force_sync_resource() { + local kind="$1" + local namespace="$2" + local name="$3" + local before_file="${work_dir}/${kind}-${namespace}-${name}-before.json" + local annotated_file="${work_dir}/${kind}-${namespace}-${name}-annotated.json" + local current_file="${work_dir}/${kind}-${namespace}-${name}-current.json" + local before_refresh + local annotated_resource_version + local attempt + local stamp + + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + get "${kind}" "${name}" \ + -o json \ + > "${before_file}" + before_refresh="$(jq -r '.status.refreshTime // ""' "${before_file}")" + stamp="$(date -u +%Y%m%dT%H%M%SZ)-$$-${RANDOM}" + + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + annotate "${kind}" "${name}" \ + "force-sync=${stamp}" \ + --overwrite \ + -o json \ + > "${annotated_file}" + annotated_resource_version="$(jq -er '.metadata.resourceVersion' \ + "${annotated_file}")" + + for ((attempt = 1; attempt <= SYNC_ATTEMPTS; attempt++)); do + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + get "${kind}" "${name}" \ + -o json \ + > "${current_file}" + if jq -e \ + --arg before "${before_refresh}" \ + --arg annotated_resource_version "${annotated_resource_version}" ' + (.status.refreshTime // "") as $refresh + | (($refresh != "" and $refresh != $before) + or ((.metadata.resourceVersion // "") != "" + and .metadata.resourceVersion != $annotated_resource_version)) + and any(.status.conditions[]?; + .type == "Ready" and .status == "True") + ' "${current_file}" >/dev/null; then + return 0 + fi + sleep "${SYNC_INTERVAL}" + done + + echo "::error::Timed out waiting for ${kind}/${namespace}/${name} to complete the forced GHCR credential sync." + return 1 +} + +# Verify that a namespace's materialized GHCR Secret matches the SOPS source. +verify_consumer_secret() { + local namespace="$1" + local secret_file="${work_dir}/consumer-${namespace}.json" + local decoded_file="${work_dir}/consumer-${namespace}-decoded.json" + local normalized_file="${work_dir}/consumer-${namespace}-normalized.json" + + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + get secret ghcr-auth \ + -o json \ + > "${secret_file}" + if ! jq -er '.data[".dockerconfigjson"] | @base64d' \ + "${secret_file}" \ + > "${decoded_file}" 2>/dev/null \ + || ! jq -S -c . "${decoded_file}" > "${normalized_file}" 2>/dev/null \ + || ! cmp -s "${expected_normalized}" "${normalized_file}"; then + echo "::error::ExternalSecret ${namespace}/ghcr-auth did not materialise the Git/SOPS GHCR credential." + return 1 + fi +} + +# Apply Git/SOPS auth to stale Talos nodes, prove an exact image pull, and only +# then record the non-secret revision marker that makes the operation retryable. +sync_talos_registry_auth() { + local desired_revision="$1" + local operator_image="ghcr.io/devantler-tech/ksail:v${KSAIL_OPERATOR_VERSION}" + local image_reference + local node_name + local node_ip + local _node_role + + if ! kubectl \ + --context "${KUBE_CONTEXT}" \ + get nodes \ + -o json \ + > "${talos_nodes_file}"; then + echo "::error::Could not list Talos nodes; refusing to mutate any Kubernetes credential consumers." + return 1 + fi + # talosctl connects to the public control-plane endpoints in talosconfig and + # proxies --nodes targets through them. Target addresses therefore must be + # the stable InternalIPs as seen by those endpoint servers, not client-facing + # ExternalIPs (Talos v1.13 "Endpoints and nodes"). + if ! jq -e ' + (.items | length) > 0 + and all(.items[]; + ([.status.addresses[]? | select(.type == "InternalIP") | .address] + | length) == 1 + and (([.status.addresses[]? + | select(.type == "InternalIP") | .address][0]) + | type == "string" and length > 0)) + and (([.items[] + | [.status.addresses[]? + | select(.type == "InternalIP") | .address][0]] + | unique | length) == (.items | length)) + ' "${talos_nodes_file}" >/dev/null; then + echo "::error::Every Talos node must expose exactly one non-empty, unique InternalIP before GHCR auth can be synchronized." + return 1 + fi + + if ! jq -r --arg revision "${desired_revision}" ' + .items[] + | select( + (.metadata.annotations[ + "platform.devantler.tech/ghcr-pull-verified-revision" + ] // "") + != $revision + ) + | (.metadata.labels // {}) as $labels + | [ + (if (($labels | has("node-role.kubernetes.io/control-plane")) + or ($labels | has("node-role.kubernetes.io/master"))) + then "1" else "0" end), + .metadata.name, + ([.status.addresses[] + | select(.type == "InternalIP") | .address][0]) + ] + | @tsv + ' "${talos_nodes_file}" \ + | LC_ALL=C sort -k1,1 -k2,2 \ + > "${talos_node_targets}"; then + echo "::error::Could not select Talos nodes requiring the GHCR auth revision." + return 1 + fi + + # Normal deploys should not regain an all-node Talos API dependency once the + # current ciphertext revision has been proved on every node. + if [[ ! -s "${talos_node_targets}" ]]; then + return 0 + fi + + if ! kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace ksail-operator \ + get deployment ksail-operator \ + --ignore-not-found \ + -o json \ + > "${ksail_operator_deployment}"; then + echo "::error::Could not determine the exact live KSail operator image." + return 1 + fi + if [[ -s "${ksail_operator_deployment}" ]]; then + if ! operator_image="$(jq -er ' + [.spec.template.spec.containers[]? | select(.name == "operator") | .image] + | if length == 1 then .[0] else error("expected one operator image") end + ' "${ksail_operator_deployment}")"; then + echo "::error::The live ksail-operator Deployment does not expose one expected first-party image." + return 1 + fi + elif [[ "${allow_incomplete_fanout}" != "true" ]]; then + echo "::error::The live ksail-operator Deployment is absent outside explicit DR bootstrap mode." + return 1 + fi + + if [[ "${operator_image}" == ghcr.io/devantler-tech/ksail:* ]]; then + image_reference="${operator_image##*:}" + if [[ -z "${image_reference}" || "${image_reference}" == "latest" ]]; then + echo "::error::The KSail operator verification image must use an exact non-latest tag or digest." + return 1 + fi + elif ! [[ "${operator_image}" =~ ^ghcr\.io/devantler-tech/ksail@sha256:[0-9a-f]{64}$ ]]; then + echo "::error::The KSail operator verification image must be the expected first-party package." + return 1 + fi + + : > "${talos_result_file}" + chmod 600 "${talos_result_file}" + while IFS=$'\t' read -r _node_role node_name node_ip; do + if ! talosctl \ + --nodes "${node_ip}" \ + patch machineconfig \ + --mode=no-reboot \ + --patch-file="${talos_auth_patch_file}" \ + >"${talos_result_file}" 2>&1; then + echo "::error::Talos node ${node_name} did not accept the Git/SOPS GHCR registry auth." + return 1 + fi + if ! talosctl \ + --nodes "${node_ip}" \ + image pull "${operator_image}" \ + --namespace cri \ + >"${talos_result_file}" 2>&1; then + echo "::error::Talos node ${node_name} could not pull the exact live KSail image after its auth refresh." + return 1 + fi + if ! talosctl \ + --nodes "${node_ip}" \ + patch machineconfig \ + --mode=no-reboot \ + --patch-file="${talos_revision_patch_file}" \ + >"${talos_result_file}" 2>&1; then + echo "::error::Talos node ${node_name} proved GHCR access but could not record the synchronized credential revision." + return 1 + fi + done < "${talos_node_targets}" +} + +# KSail embeds SOPS, so the deploy uses the same pinned toolchain as workload +# reconciliation. Decrypt only the Docker config scalar and never emit it to +# stdout or place its plaintext/base64 representation in an argument. +decrypt_flux_ghcr_docker_config "${docker_config}" "${SECRET_FILE}" +write_flux_ghcr_credentials "${docker_config}" "${credentials_file}" +jq -S -c . "${docker_config}" > "${expected_normalized}" + +# Build curl's Basic-auth config without putting the credential in argv or +# stdout. Support both Docker config representations used in this repository: +# explicit username/password and base64(username:password) in auth. +jq -r ' + "user = " + ((.username + ":" + .password) | @json) +' "${credentials_file}" > "${basic_curl_config}" +chmod 600 "${basic_curl_config}" + +# The same pull credential fans out to Flux OCI sources and private tenant +# workloads. GHCR permissions are package-granular, and the token endpoint can +# return only the intersection of requested and granted scopes. Therefore a +# token HTTP 200 is not proof of pull access: exchange it for a bearer token, +# then perform a real registry manifest GET for every package. Both credentials +# stay in mode-0600 files. --disable must remain curl's first argument so an +# ambient ~/.curlrc cannot enable tracing, add URLs, or otherwise expose auth. +for target in "${REQUIRED_PULL_TARGETS[@]}"; do + repository="${target%:*}" + reference="${target##*:}" + if ! http_status="$(curl --disable \ + --config "${basic_curl_config}" \ + --silent \ + --show-error \ + --output "${token_response}" \ + --write-out '%{http_code}' \ + --get \ + --data-urlencode 'service=ghcr.io' \ + --data-urlencode "scope=repository:${repository}:pull" \ + 'https://ghcr.io/token')"; then + echo "::error::Could not request a GHCR pull token for ${repository}; the root Flux Secret was not changed." + exit 1 + fi + if [[ "${http_status}" != "200" ]] || ! jq -e ' + (.token // .access_token // "") + | type == "string" and length > 0 + ' "${token_response}" >/dev/null; then + echo "::error::The SOPS GHCR credential could not obtain a pull token for ${repository} (GHCR HTTP ${http_status}); the root Flux Secret was not changed." + exit 1 + fi + + jq -r ' + (.token // .access_token) as $token + | "header = " + (("Authorization: Bearer " + $token) | @json) + ' "${token_response}" > "${bearer_curl_config}" + chmod 600 "${bearer_curl_config}" + + if ! http_status="$(curl --disable \ + --config "${bearer_curl_config}" \ + --silent \ + --show-error \ + --output /dev/null \ + --write-out '%{http_code}' \ + --header 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json' \ + "https://ghcr.io/v2/${repository}/manifests/${reference}")"; then + echo "::error::Could not read the GHCR manifest for ${target}; the root Flux Secret was not changed." + exit 1 + fi + if [[ "${http_status}" != "200" ]]; then + echo "::error::The SOPS GHCR pull credential cannot read ${target} (GHCR HTTP ${http_status}); the root Flux Secret was not changed." + exit 1 + fi +done + +if [[ "${check_only}" == "true" ]]; then + echo "✅ Validated every required GHCR package pull from Git/SOPS." + exit 0 +fi + +# Talos image verification resolves cosign artifacts with host registry auth; +# pod imagePullSecrets cannot satisfy that request. Use the supported v1.13 +# RegistryAuthConfig document and a file-backed patch so credentials never +# enter argv. Synchronize and verify nodes before touching Kubernetes Secrets. +jq ' + { + apiVersion: "v1alpha1", + kind: "RegistryAuthConfig", + name: "ghcr.io", + username: .username, + password: .password + } +' "${credentials_file}" > "${talos_auth_patch_file}" +pull_revision="$(flux_ghcr_revision "${SECRET_FILE}")" +readonly pull_revision +jq -n --arg revision "${pull_revision}" ' + { + machine: { + nodeAnnotations: { + "platform.devantler.tech/ghcr-pull-verified-revision": $revision + } + } + } +' > "${talos_revision_patch_file}" +chmod 600 "${talos_auth_patch_file}" "${talos_revision_patch_file}" +sync_talos_registry_auth "${pull_revision}" + +# Merge only Secret data fields so ownership metadata survives. The sensitive +# payload stays in pipes/temp files and never appears in argv or logs. +base64 < "${docker_config}" \ + | tr -d '\r\n' \ + | jq -Rs '{data: {".dockerconfigjson": .}}' \ + > "${patch_file}" + +# Patch only the root Flux Secret payload, preserving KSail ownership metadata. +patch_root_secret() { + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + patch secret ksail-registry-credentials \ + --type=merge \ + --patch-file="${patch_file}" +} + +patch_variables_base() { + kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + patch secret variables-base \ + --type=merge \ + --patch-file="${variables_patch_file}" +} + +# A fresh DR cluster does not have variables-base or the ESO fan-out resources +# until its first Flux reconcile. In that case the current artifact creates the +# chain from the same SOPS value, so only the root bootstrap patch is needed. +if ! variables_base_name="$(kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + get secret variables-base \ + --ignore-not-found \ + -o name)"; then + echo "::error::Could not determine whether the GHCR fan-out exists; refusing to reconcile with an unverified tenant credential path." + exit 1 +fi +if [[ -z "${variables_base_name}" ]]; then + if [[ "${allow_incomplete_fanout}" != "true" ]]; then + echo "::error::The GHCR fan-out is not initialized; root Flux auth was not changed. Use --allow-incomplete-fanout only during the DR bootstrap, then run the full verifier after reconciliation." + exit 1 + fi + patch_root_secret + echo "✅ Refreshed root Flux GHCR auth; the first reconcile will create the downstream fan-out." + exit 0 +fi + +# Prepare the variables-base payload locally, but do not mutate its live Secret +# until normal mode has proved the complete fan-out exists. Otherwise a failed +# normal deploy could leave PushSecret free to propagate an unmerged credential +# even though root Flux auth stayed unchanged. +jq '{data: {ghcr_dockerconfigjson: .data[".dockerconfigjson"]}}' \ + "${patch_file}" \ + > "${variables_patch_file}" + +# A partially-bootstrapped DR cluster can already have variables-base while ESO +# CRDs or individual fan-out objects do not exist yet. That state still needs +# root auth so Flux can fetch the artifact that completes the chain. Distinguish +# an absent API/resource from a failed lookup, and never force-sync a partial set. +if ! kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + api-resources \ + --api-group=external-secrets.io \ + -o name \ + > "${fanout_api_resources}"; then + echo "::error::Could not inspect the External Secrets API; refusing to change root Flux auth." + exit 1 +fi + +fanout_complete=true +if ! grep -qx 'pushsecrets.external-secrets.io' "${fanout_api_resources}" \ + || ! grep -qx 'externalsecrets.external-secrets.io' "${fanout_api_resources}"; then + fanout_complete=false +else + if ! pushsecret_name="$(kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace flux-system \ + get pushsecret seed-ghcr \ + --ignore-not-found \ + -o name)"; then + echo "::error::Could not determine whether PushSecret flux-system/seed-ghcr exists; refusing to change root Flux auth." + exit 1 + fi + if [[ -z "${pushsecret_name}" ]]; then + fanout_complete=false + fi + + for namespace in "${FANOUT_NAMESPACES[@]}"; do + if ! externalsecret_name="$(kubectl \ + --context "${KUBE_CONTEXT}" \ + --namespace "${namespace}" \ + get externalsecret ghcr-auth \ + --ignore-not-found \ + -o name)"; then + echo "::error::Could not determine whether ExternalSecret ${namespace}/ghcr-auth exists; refusing to change root Flux auth." + exit 1 + fi + if [[ -z "${externalsecret_name}" ]]; then + fanout_complete=false + fi + done +fi + +if [[ "${fanout_complete}" != "true" ]]; then + if [[ "${allow_incomplete_fanout}" != "true" ]]; then + echo "::error::The GHCR fan-out is incomplete; root Flux auth was not changed. Use --allow-incomplete-fanout only during the DR bootstrap, then run the full verifier after reconciliation." + exit 1 + fi + patch_root_secret + patch_variables_base + patch_root_secret + echo "✅ Staged the Git/SOPS credential and refreshed root Flux auth; the first reconcile will complete the missing downstream fan-out." + exit 0 +fi + +# Existing clusters update and verify the whole SOPS -> variables-base -> +# PushSecret -> OpenBao -> ExternalSecret chain before switching root Flux auth. +patch_variables_base +force_sync_resource pushsecret flux-system seed-ghcr +for namespace in "${FANOUT_NAMESPACES[@]}"; do + force_sync_resource externalsecret "${namespace}" ghcr-auth + verify_consumer_secret "${namespace}" +done + +patch_root_secret + +echo "✅ Synchronised every existing consumer and refreshed root Flux GHCR auth from Git/SOPS." diff --git a/scripts/run-ksail-prod-with-pull-auth.sh b/scripts/run-ksail-prod-with-pull-auth.sh new file mode 100755 index 000000000..5840225d2 --- /dev/null +++ b/scripts/run-ksail-prod-with-pull-auth.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Run credential-consuming KSail lifecycle commands with Git/SOPS pull auth. + +set -euo pipefail +set +x + +case "${1:-} ${2:-}" in + "cluster create" | "cluster update" | "workload push" | "workload reconcile") ;; + *) + echo "Usage: $0 {cluster create|cluster update|workload push|workload reconcile}" >&2 + exit 64 + ;; +esac +if (($# != 2)); then + echo "Usage: $0 {cluster create|cluster update|workload push|workload reconcile}" >&2 + exit 64 +fi + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +# shellcheck source=scripts/ghcr-auth-lib.sh +source "${SCRIPT_DIR}/ghcr-auth-lib.sh" +require_flux_ghcr_yaml_tool + +readonly SECRET_FILE="${FLUX_GHCR_SECRET_FILE:-k8s/bases/bootstrap/secret.enc.yaml}" +# Viper applies this nested config override before KSail expands the two child +# environment variables. The template itself contains no credential. +# shellcheck disable=SC2016 +readonly PULL_REGISTRY_TEMPLATE='${GHCR_USERNAME}:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests' +pull_revision="$(flux_ghcr_revision "${SECRET_FILE}")" +readonly pull_revision + +# A shell variable is required to pass the token in the child environment. +# It never enters argv or stdout, and the constrained command set prevents an +# arbitrary process from inheriting it. The Actions GHCR_TOKEN remains in the +# workload-push step; only lifecycle reads override it. The non-secret SOPS +# ciphertext revision defeats KSail's deliberate credential redaction in +# machine-config fingerprints, so autoscaler templates still refresh on +# token-only rotations. +if [[ "$1 $2" == "workload push" ]]; then + if [[ -z "${GHCR_TOKEN:-}" || -z "${GITHUB_ACTOR:-}" ]]; then + echo "::error::The Actions GHCR publish token and actor are required for workload push." + exit 1 + fi + GHCR_USERNAME="${GITHUB_ACTOR}" \ + GHCR_PULL_REVISION="${pull_revision}" \ + ksail --config ksail.prod.yaml "$1" "$2" +else + work_dir="$(mktemp -d)" + trap 'rm -rf "${work_dir}"' EXIT + chmod 700 "${work_dir}" + umask 077 + + docker_config="${work_dir}/config.json" + credentials_file="${work_dir}/credentials.json" + decrypt_flux_ghcr_docker_config "${docker_config}" "${SECRET_FILE}" + write_flux_ghcr_credentials "${docker_config}" "${credentials_file}" + pull_username="$(jq -er '.username' "${credentials_file}")" + pull_token="$(jq -er '.password' "${credentials_file}")" + KSAIL_SPEC_CLUSTER_LOCALREGISTRY_REGISTRY="${PULL_REGISTRY_TEMPLATE}" \ + GHCR_USERNAME="${pull_username}" \ + GHCR_TOKEN="${pull_token}" \ + GHCR_PULL_REVISION="${pull_revision}" \ + ksail --config ksail.prod.yaml "$1" "$2" +fi diff --git a/scripts/tests/test_refresh_flux_ghcr_auth.py b/scripts/tests/test_refresh_flux_ghcr_auth.py new file mode 100644 index 000000000..3929c7829 --- /dev/null +++ b/scripts/tests/test_refresh_flux_ghcr_auth.py @@ -0,0 +1,1639 @@ +"""Regression tests for the production Flux GHCR credential bridge.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +AGENT_INSTRUCTIONS = ROOT / "AGENTS.md" +HELPER = ROOT / "scripts" / "refresh-flux-ghcr-auth.sh" +GHCR_AUTH_LIB = ROOT / "scripts" / "ghcr-auth-lib.sh" +KSAIL_PULL_WRAPPER = ROOT / "scripts" / "run-ksail-prod-with-pull-auth.sh" +KSAIL_PROD_CONFIG = ROOT / "ksail.prod.yaml" +ACTION = ROOT / ".github" / "actions" / "deploy-prod" / "action.yml" +DR_REBUILD = ROOT / ".github" / "workflows" / "dr-rebuild.yaml" +CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yaml" +DR_RUNBOOK = ROOT / "docs" / "dr" / "runbook.md" +KSAIL_OPERATOR_HELM_RELEASE = ( + ROOT + / "k8s" + / "bases" + / "infrastructure" + / "controllers" + / "ksail-operator" + / "helm-release.yaml" +) +KSAIL_OPERATOR_VERSION = next( + line.split(":", 1)[1].strip() + for line in KSAIL_OPERATOR_HELM_RELEASE.read_text(encoding="utf-8").splitlines() + if line.strip().startswith("version:") +) +PROVIDER_UPJET_UNIFI = ( + ROOT + / "k8s" + / "providers" + / "hetzner" + / "infrastructure" + / "crossplane" + / "provider-upjet-unifi.yaml" +) +TALOS_GHCR_AUTH = ROOT / "talos" / "cluster" / "authenticate-ghcr-pulls.yaml" +TALOS_GHCR_REVISION = ( + ROOT / "talos" / "cluster" / "mark-ghcr-pull-revision.yaml" +) + + +class RefreshFluxGhcrAuthTests(unittest.TestCase): + """Exercise the helper with fake external commands and no real secrets.""" + + def setUp(self) -> None: + """Create isolated command fakes and capture files for each test.""" + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + self.workspace = Path(self.temp_dir.name) + self.bin_dir = self.workspace / "bin" + self.bin_dir.mkdir() + self.decrypted_config = self.workspace / "decrypted-config.json" + self.encrypted_secret = self.workspace / "secret.enc.yaml" + self.patch_capture = self.workspace / "patch.json" + self.variables_patch_capture = self.workspace / "variables-patch.json" + self.kubectl_called = self.workspace / "kubectl-called" + self.output_path_log = self.workspace / "ksail-output-path" + self.registry_read_log = self.workspace / "registry-reads" + self.fanout_log = self.workspace / "fanout-log" + self.talos_log = self.workspace / "talos-log" + self.talos_patch_path_log = self.workspace / "talos-patch-path" + self.operation_log = self.workspace / "operation-log" + self.ksail_token_capture = self.workspace / "ksail-token" + self.ksail_username_capture = self.workspace / "ksail-username" + self.ksail_revision_capture = self.workspace / "ksail-revision" + self.ksail_command_capture = self.workspace / "ksail-command" + self.ksail_config_path_capture = self.workspace / "ksail-config-path" + self.ksail_registry_capture = self.workspace / "ksail-registry" + self.ksail_registry_override_capture = ( + self.workspace / "ksail-registry-override" + ) + self.sync_state_dir = self.workspace / "sync-state" + self.sync_state_dir.mkdir() + self._write_encrypted_secret("ENC[AES256_GCM,data:fixture-one]") + + self._write_executable( + "ksail", + """ + #!/usr/bin/env bash + set -euo pipefail + arguments=" $* " + selector='["stringData"]["ghcr_dockerconfigjson"]' + if [[ "$arguments" == *" workload cipher decrypt "* ]]; then + [[ "$arguments" == *" --extract $selector "* ]] + output="" + while (($#)); do + case "$1" in + --output) + output="$2" + shift 2 + ;; + *) shift ;; + esac + done + test -n "$output" + printf '%s' "$output" > "$KSAIL_OUTPUT_PATH_LOG" + cp "$FAKE_DECRYPTED_CONFIG" "$output" + exit 0 + fi + + if [[ "$arguments" == *" cluster create "* \ + || "$arguments" == *" cluster update "* \ + || "$arguments" == *" workload push "* \ + || "$arguments" == *" workload reconcile "* ]]; then + config="" + previous="" + for argument in "$@"; do + if [[ "$previous" == --config ]]; then + config="$argument" + break + fi + previous="$argument" + done + test -f "$config" + registry="$(yq -er '.spec.cluster.localRegistry.registry' "$config")" + test "$registry" = \ + 'devantler:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests' + registry_override="${KSAIL_SPEC_CLUSTER_LOCALREGISTRY_REGISTRY:-}" + if [[ "$arguments" == *" workload push "* ]]; then + test -z "$registry_override" + else + test "$registry_override" = \ + '${GHCR_USERNAME}:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests' + fi + test -n "${GHCR_TOKEN:-}" + test -n "${GHCR_USERNAME:-}" + test "${GHCR_PULL_REVISION:-}" != "" + printf '%s' "$GHCR_TOKEN" > "$KSAIL_TOKEN_CAPTURE" + printf '%s' "$GHCR_USERNAME" > "$KSAIL_USERNAME_CAPTURE" + printf '%s' "$GHCR_PULL_REVISION" > "$KSAIL_REVISION_CAPTURE" + printf '%s\n' "$*" > "$KSAIL_COMMAND_CAPTURE" + printf '%s' "$config" > "$KSAIL_CONFIG_PATH_CAPTURE" + printf '%s' "$registry" > "$KSAIL_REGISTRY_CAPTURE" + printf '%s' "$registry_override" > "$KSAIL_REGISTRY_OVERRIDE_CAPTURE" + exit 0 + fi + + echo "unexpected ksail invocation" >&2 + exit 92 + """, + ) + self._write_executable( + "talosctl", + """ + #!/usr/bin/env bash + set -euo pipefail + arguments=" $* " + node="" + patch_file="" + previous="" + for argument in "$@"; do + if [[ "$previous" == --nodes ]]; then + node="$argument" + fi + case "$argument" in + --nodes=*) node="${argument#*=}" ;; + --patch-file=*) patch_file="${argument#*=}" ;; + esac + previous="$argument" + done + test -n "$node" + + if [[ "$arguments" == *" patch machineconfig "* ]]; then + [[ "$arguments" == *" --mode=no-reboot "* ]] + test -f "$patch_file" + printf '%s' "$patch_file" > "$TALOS_PATCH_PATH_LOG" + if jq -e '.kind == "RegistryAuthConfig"' "$patch_file" >/dev/null; then + jq -e \ + --arg username "$EXPECTED_PULL_USERNAME" \ + --arg token "$EXPECTED_PULL_TOKEN" ' + .apiVersion == "v1alpha1" + and .kind == "RegistryAuthConfig" + and .name == "ghcr.io" + and .username == $username + and .password == $token + ' "$patch_file" >/dev/null + printf 'talos-auth:%s\n' "$node" >> "$TALOS_LOG" + printf 'talos-auth:%s\n' "$node" >> "$OPERATION_LOG" + if [[ "$node" == "${FAKE_TALOS_FAIL_NODE:-disabled}" \ + && "${FAKE_TALOS_FAIL_OPERATION:-auth}" == auth ]]; then + echo "talos auth failed with $EXPECTED_PULL_TOKEN" >&2 + exit 45 + fi + touch "$FAKE_SYNC_STATE_DIR/talos-auth-${node}" + exit 0 + fi + + test -f "$FAKE_SYNC_STATE_DIR/talos-auth-${node}" + test -f "$FAKE_SYNC_STATE_DIR/talos-pull-${node}" + jq -e --arg revision "$EXPECTED_GHCR_REVISION" ' + .machine.nodeAnnotations[ + "platform.devantler.tech/ghcr-pull-verified-revision" + ] == $revision + ' "$patch_file" >/dev/null + printf 'talos-revision:%s\n' "$node" >> "$TALOS_LOG" + printf 'talos-revision:%s\n' "$node" >> "$OPERATION_LOG" + if [[ "$node" == "${FAKE_TALOS_FAIL_NODE:-disabled}" \ + && "${FAKE_TALOS_FAIL_OPERATION:-auth}" == revision ]]; then + echo "talos revision failed" >&2 + exit 48 + fi + touch "$FAKE_SYNC_STATE_DIR/talos-revision-${node}" + exit 0 + fi + + if [[ "$arguments" == *" image pull "* ]]; then + test -f "$FAKE_SYNC_STATE_DIR/talos-auth-${node}" + [[ "$arguments" == *" --namespace cri "* ]] + image="" + previous="" + for argument in "$@"; do + if [[ "$previous" == pull ]]; then + image="$argument" + break + fi + previous="$argument" + done + test -n "$image" + printf 'talos-pull:%s:%s\n' "$node" "$image" >> "$TALOS_LOG" + printf 'talos-pull:%s:%s\n' "$node" "$image" >> "$OPERATION_LOG" + if [[ "$node" == "${FAKE_TALOS_FAIL_NODE:-disabled}" \ + && "${FAKE_TALOS_FAIL_OPERATION:-auth}" == pull ]]; then + echo "talos pull failed with $EXPECTED_PULL_TOKEN" >&2 + exit 47 + fi + touch "$FAKE_SYNC_STATE_DIR/talos-pull-${node}" + exit 0 + fi + + echo "unexpected talosctl invocation" >&2 + exit 93 + """, + ) + self._write_executable( + "curl", + """ + #!/usr/bin/env bash + set -euo pipefail + test "$1" = --disable + shift + config="" + output="" + scope="" + url="" + while (($#)); do + case "$1" in + --config) + config="$2" + shift 2 + ;; + --output) + output="$2" + shift 2 + ;; + --data-urlencode) + case "$2" in scope=*) scope="${2#scope=}" ;; esac + shift 2 + ;; + --write-out|--header) + shift 2 + ;; + --silent|--show-error|--get) + shift + ;; + https://*) + url="$1" + shift + ;; + *) + echo "unexpected curl argument: $1" >&2 + exit 90 + ;; + esac + done + test -f "$config" + test -n "$output" + test -n "$url" + + if [[ "$url" == https://ghcr.io/token ]]; then + test -n "$scope" + grep -q '^user = ' "$config" + printf '{"token":"fixture-registry-token"}' > "$output" + printf '200' + exit 0 + fi + + manifest_path="${url#https://ghcr.io/v2/}" + repository="${manifest_path%/manifests/*}" + reference="${manifest_path##*/manifests/}" + test "$repository" != "$manifest_path" + test "$reference" != "$manifest_path" + grep -q 'Authorization: Bearer fixture-registry-token' "$config" + printf '%s:%s\n' "$repository" "$reference" >> "$REGISTRY_READ_LOG" + if [[ "$repository" == "${FAKE_CURL_DENY_REPOSITORY:-disabled}" ]]; then + printf '403' + else + printf '200' + fi + """, + ) + self._write_executable( + "kubectl", + """ + #!/usr/bin/env bash + set -euo pipefail + arguments=" $* " + [[ "$arguments" == *" --context admin@prod "* ]] + namespace="" + patch_file="" + previous="" + for argument in "$@"; do + if [[ "$previous" == --namespace ]]; then + namespace="$argument" + fi + case "$argument" in + --namespace=*) namespace="${argument#*=}" ;; + --patch-file=*) patch_file="${argument#*=}" ;; + esac + previous="$argument" + done + touch "$KUBECTL_CALLED" + + if [[ "$arguments" == *" get nodes "* ]]; then + if [[ "${FAKE_NODE_DISCOVERY_FAIL:-false}" == true ]]; then + echo 'node discovery failed' >&2 + exit 46 + fi + if [[ -n "${FAKE_NODE_JSON:-}" ]]; then + printf '%s\n' "$FAKE_NODE_JSON" + exit 0 + fi + jq -n \ + --arg revision "$EXPECTED_GHCR_REVISION" \ + --arg current "${FAKE_TALOS_NODES_CURRENT:-false}" ' + { + items: [ + { + metadata: { + name: "prod-worker-1", + labels: {}, + annotations: { + "platform.devantler.tech/ghcr-pull-desired-revision": + $revision + } + }, + status: {addresses: [ + {type: "InternalIP", address: "10.0.0.2"}, + {type: "ExternalIP", address: "198.51.100.2"} + ]} + }, + { + metadata: { + name: "prod-control-plane-1", + labels: { + "node-role.kubernetes.io/control-plane": "" + }, + annotations: { + "platform.devantler.tech/ghcr-pull-desired-revision": + $revision + } + }, + status: {addresses: [ + {type: "InternalIP", address: "10.0.0.1"}, + {type: "ExternalIP", address: "198.51.100.1"} + ]} + } + ] + } + | if $current == "true" then + .items[].metadata.annotations[ + "platform.devantler.tech/ghcr-pull-verified-revision" + ] = $revision + else . end + ' + exit 0 + fi + + test -n "$namespace" + + if [[ "$arguments" == *" get deployment ksail-operator "* ]]; then + test "$namespace" = ksail-operator + if [[ "${FAKE_KSAIL_OPERATOR_DEPLOYMENT_ABSENT:-false}" == true ]]; then + exit 0 + fi + printf '{"spec":{"template":{"spec":{"containers":[{"name":"sidecar","image":"public.example/sidecar:latest"},{"name":"operator","image":"%s"}]}}}}\n' \ + "${FAKE_KSAIL_OPERATOR_IMAGE:-ghcr.io/devantler-tech/ksail:v7.169.0}" + exit 0 + fi + + if [[ "$arguments" == *" api-resources "* ]]; then + [[ "$arguments" == *" --api-group=external-secrets.io "* ]] + if [[ "${FAKE_FANOUT_CRDS_ABSENT:-false}" != true ]]; then + printf '%s\n' \ + 'externalsecrets.external-secrets.io' \ + 'pushsecrets.external-secrets.io' + fi + exit 0 + fi + + if [[ "$arguments" == *" patch secret ksail-registry-credentials "* ]]; then + [[ "$arguments" == *" --type=merge "* ]] + test -n "$patch_file" + cp "$patch_file" "$PATCH_CAPTURE" + printf 'root-patch\n' >> "$OPERATION_LOG" + if [[ "${FAKE_KUBECTL_FAIL:-false}" == true ]]; then + echo 'cluster patch failed' >&2 + exit 43 + fi + echo 'secret/ksail-registry-credentials patched' + exit 0 + fi + + if [[ "$arguments" == *" get secret variables-base "* ]]; then + [[ "$arguments" == *" --ignore-not-found "* ]] + if [[ "${FAKE_VARIABLES_BASE_ABSENT:-false}" != true ]]; then + echo 'secret/variables-base' + fi + exit 0 + fi + + if [[ "$arguments" == *" patch secret variables-base "* ]]; then + [[ "$arguments" == *" --type=merge "* ]] + test -n "$patch_file" + cp "$patch_file" "$VARIABLES_PATCH_CAPTURE" + printf 'variables-patch\n' >> "$OPERATION_LOG" + echo 'secret/variables-base patched' + exit 0 + fi + + kind="" + name="" + if [[ "$arguments" == *" pushsecret seed-ghcr "* ]]; then + kind=pushsecret + name=seed-ghcr + elif [[ "$arguments" == *" externalsecret ghcr-auth "* ]]; then + kind=externalsecret + name=ghcr-auth + fi + + if [[ -n "$kind" && "$arguments" == *" get $kind $name "* \ + && "$arguments" == *" --ignore-not-found "* ]]; then + resource="$kind/$namespace/$name" + if [[ "$resource" != "${FAKE_MISSING_FANOUT_RESOURCE:-disabled}" ]]; then + echo "$kind/$name" + fi + exit 0 + fi + + if [[ -n "$kind" \ + && "$kind/$namespace/$name" == "${FAKE_MISSING_FANOUT_RESOURCE:-disabled}" ]]; then + echo "$kind/$name not found" >&2 + exit 44 + fi + + if [[ -n "$kind" && "$arguments" == *" get $kind $name "* ]]; then + marker="$FAKE_SYNC_STATE_DIR/${kind}-${namespace}-${name}" + annotated_marker="${marker}-annotated" + refresh_time=2026-07-13T00:00:00Z + resource_version=1 + if [[ -f "$annotated_marker" ]]; then + resource_version=2 + fi + if [[ -f "$marker" && "${FAKE_SYNC_SAME_REFRESH_TIME:-false}" != true ]]; then + refresh_time=2026-07-13T00:00:01Z + fi + if [[ -f "$marker" ]]; then + resource_version=3 + fi + printf '{"metadata":{"resourceVersion":"%s"},"status":{"refreshTime":"%s","conditions":[{"type":"Ready","status":"True"}]}}\n' "$resource_version" "$refresh_time" + exit 0 + fi + + if [[ -n "$kind" && "$arguments" == *" annotate $kind $name "* ]]; then + resource="$kind/$namespace/$name" + printf '%s\n' "$resource" >> "$FANOUT_LOG" + printf 'fanout:%s\n' "$resource" >> "$OPERATION_LOG" + marker="$FAKE_SYNC_STATE_DIR/${kind}-${namespace}-${name}" + touch "${marker}-annotated" + if [[ "$resource" != "${FAKE_SYNC_STALL_RESOURCE:-disabled}" ]]; then + touch "$marker" + fi + printf '{"metadata":{"resourceVersion":"2"}}\n' + exit 0 + fi + + if [[ "$arguments" == *" get secret ghcr-auth "* ]]; then + if [[ "$namespace" == "${FAKE_CONSUMER_MISMATCH_NAMESPACE:-disabled}" ]]; then + encoded=$(printf '%s' '{"auths":{}}' | base64 | tr -d '\r\n') + else + encoded=$(jq -r '.data.ghcr_dockerconfigjson' "$VARIABLES_PATCH_CAPTURE") + fi + jq -n --arg encoded "$encoded" '{data:{".dockerconfigjson":$encoded}}' + exit 0 + fi + + echo "unexpected kubectl invocation: $arguments" >&2 + exit 91 + """, + ) + + def _write_executable(self, name: str, body: str) -> None: + """Install an executable command fake in the isolated test PATH.""" + path = self.bin_dir / name + path.write_text(textwrap.dedent(body).lstrip(), encoding="utf-8") + path.chmod(0o755) + + def _write_encrypted_secret(self, ciphertext: str) -> None: + """Write a non-secret SOPS ciphertext fixture for revision tests.""" + self.encrypted_ciphertext = ciphertext + self.encrypted_secret.write_text( + json.dumps( + {"stringData": {"ghcr_dockerconfigjson": ciphertext}} + ), + encoding="utf-8", + ) + + @staticmethod + def _expected_credentials(config: object) -> tuple[str, str]: + """Extract expected credentials from a valid Docker config fixture.""" + try: + registry = config["auths"]["ghcr.io"] # type: ignore[index] + username = registry.get("username") + password = registry.get("password") + if username and password: + return str(username), str(password) + decoded = base64.b64decode(registry["auth"]).decode() + decoded_username, decoded_password = decoded.split(":", 1) + return decoded_username, decoded_password + except (KeyError, TypeError, ValueError): + return "unused", "unused" + + def _expected_revision(self) -> str: + """Match the helper's SHA-256 of the yq-emitted ciphertext scalar.""" + return hashlib.sha256( + f"{self.encrypted_ciphertext}\n".encode() + ).hexdigest() + + def _run_helper( + self, + config: object, + helper_args: tuple[str, ...] = (), + **environment_overrides: str, + ) -> subprocess.CompletedProcess[str]: + """Run the credential bridge against a supplied Docker config fixture.""" + self.decrypted_config.write_text(json.dumps(config), encoding="utf-8") + for marker in ( + self.patch_capture, + self.variables_patch_capture, + self.kubectl_called, + self.output_path_log, + self.registry_read_log, + self.fanout_log, + self.talos_log, + self.talos_patch_path_log, + self.operation_log, + self.ksail_token_capture, + self.ksail_username_capture, + self.ksail_revision_capture, + self.ksail_command_capture, + self.ksail_config_path_capture, + self.ksail_registry_capture, + self.ksail_registry_override_capture, + ): + marker.unlink(missing_ok=True) + for marker in self.sync_state_dir.iterdir(): + marker.unlink() + expected_username, expected_token = self._expected_credentials(config) + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.bin_dir}:{environment['PATH']}", + "FAKE_DECRYPTED_CONFIG": str(self.decrypted_config), + "FLUX_GHCR_SECRET_FILE": str(self.encrypted_secret), + "PATCH_CAPTURE": str(self.patch_capture), + "VARIABLES_PATCH_CAPTURE": str(self.variables_patch_capture), + "KUBECTL_CALLED": str(self.kubectl_called), + "KSAIL_OUTPUT_PATH_LOG": str(self.output_path_log), + "REGISTRY_READ_LOG": str(self.registry_read_log), + "FANOUT_LOG": str(self.fanout_log), + "TALOS_LOG": str(self.talos_log), + "TALOS_PATCH_PATH_LOG": str(self.talos_patch_path_log), + "OPERATION_LOG": str(self.operation_log), + "KSAIL_TOKEN_CAPTURE": str(self.ksail_token_capture), + "KSAIL_USERNAME_CAPTURE": str(self.ksail_username_capture), + "KSAIL_REVISION_CAPTURE": str(self.ksail_revision_capture), + "KSAIL_COMMAND_CAPTURE": str(self.ksail_command_capture), + "KSAIL_CONFIG_PATH_CAPTURE": str(self.ksail_config_path_capture), + "KSAIL_REGISTRY_CAPTURE": str(self.ksail_registry_capture), + "KSAIL_REGISTRY_OVERRIDE_CAPTURE": str( + self.ksail_registry_override_capture + ), + "EXPECTED_PULL_USERNAME": expected_username, + "EXPECTED_PULL_TOKEN": expected_token, + "EXPECTED_GHCR_REVISION": self._expected_revision(), + "FAKE_SYNC_STATE_DIR": str(self.sync_state_dir), + "FLUX_GHCR_SYNC_ATTEMPTS": "2", + "FLUX_GHCR_SYNC_INTERVAL": "0", + } + ) + environment.update(environment_overrides) + return subprocess.run( + [str(HELPER), *helper_args], + cwd=ROOT, + env=environment, + text=True, + capture_output=True, + check=False, + ) + + def _run_ksail_pull_wrapper( + self, + config: object, + command: tuple[str, ...], + **environment_overrides: str, + ) -> subprocess.CompletedProcess[str]: + """Run a production KSail command through the SOPS pull-auth wrapper.""" + self.decrypted_config.write_text(json.dumps(config), encoding="utf-8") + for marker in ( + self.output_path_log, + self.ksail_token_capture, + self.ksail_username_capture, + self.ksail_revision_capture, + self.ksail_command_capture, + self.ksail_config_path_capture, + self.ksail_registry_capture, + self.ksail_registry_override_capture, + ): + marker.unlink(missing_ok=True) + expected_username, expected_token = self._expected_credentials(config) + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.bin_dir}:{environment['PATH']}", + "FAKE_DECRYPTED_CONFIG": str(self.decrypted_config), + "FLUX_GHCR_SECRET_FILE": str(self.encrypted_secret), + "KSAIL_OUTPUT_PATH_LOG": str(self.output_path_log), + "KSAIL_TOKEN_CAPTURE": str(self.ksail_token_capture), + "KSAIL_USERNAME_CAPTURE": str(self.ksail_username_capture), + "KSAIL_REVISION_CAPTURE": str(self.ksail_revision_capture), + "KSAIL_COMMAND_CAPTURE": str(self.ksail_command_capture), + "KSAIL_CONFIG_PATH_CAPTURE": str(self.ksail_config_path_capture), + "KSAIL_REGISTRY_CAPTURE": str(self.ksail_registry_capture), + "KSAIL_REGISTRY_OVERRIDE_CAPTURE": str( + self.ksail_registry_override_capture + ), + "EXPECTED_PULL_USERNAME": expected_username, + "EXPECTED_PULL_TOKEN": expected_token, + "EXPECTED_GHCR_REVISION": self._expected_revision(), + } + ) + environment.update(environment_overrides) + return subprocess.run( + [str(KSAIL_PULL_WRAPPER), *command], + cwd=ROOT, + env=environment, + text=True, + capture_output=True, + check=False, + ) + + @staticmethod + def _valid_config() -> dict[str, object]: + """Return a valid explicit GHCR Docker authentication fixture.""" + return { + "auths": { + "ghcr.io": { + "username": "devantler", + "password": "fixture-secret-token", + } + } + } + + def test_refreshes_root_and_fanout_without_leaking_plaintext(self) -> None: + """Refresh every credential consumer without exposing the pull token.""" + config = self._valid_config() + + result = self._run_helper(config) + + self.assertEqual(result.returncode, 0, result.stderr) + patch = json.loads(self.patch_capture.read_text(encoding="utf-8")) + encoded = patch["data"][".dockerconfigjson"] + + self.assertEqual(json.loads(base64.b64decode(encoded)), config) + variables_patch = json.loads( + self.variables_patch_capture.read_text(encoding="utf-8") + ) + variables_encoded = variables_patch["data"]["ghcr_dockerconfigjson"] + self.assertEqual(json.loads(base64.b64decode(variables_encoded)), config) + temporary_config = Path(self.output_path_log.read_text(encoding="utf-8")) + self.assertFalse(temporary_config.exists()) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + self.assertEqual( + self.registry_read_log.read_text(encoding="utf-8").splitlines(), + [ + "devantler-tech/platform/manifests:latest", + "devantler-tech/wedding-app/manifests:latest", + "devantler-tech/ascoachingogvaner/manifests:latest", + "devantler-tech/wedding-app:latest", + "devantler-tech/ascoachingogvaner:latest", + f"devantler-tech/ksail:v{KSAIL_OPERATOR_VERSION}", + "devantler-tech/provider-upjet-unifi:v0.1.0", + ], + ) + self.assertEqual( + self.fanout_log.read_text(encoding="utf-8").splitlines(), + [ + "pushsecret/flux-system/seed-ghcr", + "externalsecret/wedding-app/ghcr-auth", + "externalsecret/ascoachingogvaner/ghcr-auth", + "externalsecret/kyverno/ghcr-auth", + ], + ) + + def test_refreshes_talos_auth_before_kubernetes_credential_consumers(self) -> None: + """Patch nodes serially and pull the exact live operator image first.""" + result = self._run_helper(self._valid_config()) + + self.assertEqual(result.returncode, 0, result.stderr) + expected_talos_operations = [ + "talos-auth:10.0.0.2", + "talos-pull:10.0.0.2:ghcr.io/devantler-tech/ksail:v7.169.0", + "talos-revision:10.0.0.2", + "talos-auth:10.0.0.1", + "talos-pull:10.0.0.1:ghcr.io/devantler-tech/ksail:v7.169.0", + "talos-revision:10.0.0.1", + ] + self.assertEqual( + self.talos_log.read_text(encoding="utf-8").splitlines(), + expected_talos_operations, + ) + self.assertEqual( + self.operation_log.read_text(encoding="utf-8").splitlines()[:6], + expected_talos_operations, + ) + temporary_patch = Path( + self.talos_patch_path_log.read_text(encoding="utf-8") + ) + self.assertFalse(temporary_patch.exists()) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + + def test_talos_failure_prevents_kubernetes_credential_mutation(self) -> None: + """Stop before Flux fan-out when any node cannot accept or use auth.""" + for operation in ("auth", "pull", "revision"): + with self.subTest(operation=operation): + result = self._run_helper( + self._valid_config(), + FAKE_TALOS_FAIL_NODE="10.0.0.2", + FAKE_TALOS_FAIL_OPERATION=operation, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + + def test_current_talos_nodes_skip_talos_api(self) -> None: + """Avoid a Talos availability dependency after this revision is proved.""" + result = self._run_helper( + self._valid_config(), + FAKE_TALOS_NODES_CURRENT="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(self.talos_log.exists()) + self.assertTrue(self.patch_capture.exists()) + + def test_dr_without_operator_uses_exact_declared_image(self) -> None: + """Verify stale bootstrap nodes before the operator Deployment exists.""" + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_KSAIL_OPERATOR_DEPLOYMENT_ABSENT="true", + FAKE_VARIABLES_BASE_ABSENT="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + pulls = [ + line + for line in self.talos_log.read_text(encoding="utf-8").splitlines() + if line.startswith("talos-pull:") + ] + self.assertEqual( + pulls, + [ + "talos-pull:10.0.0.2:ghcr.io/devantler-tech/ksail:" + f"v{KSAIL_OPERATOR_VERSION}", + "talos-pull:10.0.0.1:ghcr.io/devantler-tech/ksail:" + f"v{KSAIL_OPERATOR_VERSION}", + ], + ) + + def test_invalid_node_inventory_fails_closed(self) -> None: + """Reject empty, duplicate, and ambiguous Talos node InternalIPs.""" + invalid_inventories = [ + {"items": []}, + { + "items": [ + { + "metadata": {"name": "one"}, + "status": {"addresses": []}, + } + ] + }, + { + "items": [ + { + "metadata": {"name": "one"}, + "status": { + "addresses": [ + {"type": "InternalIP", "address": "10.0.0.1"}, + {"type": "InternalIP", "address": "10.0.0.2"}, + ] + }, + } + ] + }, + { + "items": [ + { + "metadata": {"name": "one"}, + "status": {"addresses": [ + {"type": "InternalIP", "address": "10.0.0.1"} + ]}, + }, + { + "metadata": {"name": "two"}, + "status": {"addresses": [ + {"type": "InternalIP", "address": "10.0.0.1"} + ]}, + }, + ] + }, + ] + for inventory in invalid_inventories: + with self.subTest(inventory=inventory): + result = self._run_helper( + self._valid_config(), + FAKE_NODE_JSON=json.dumps(inventory), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.talos_log.exists()) + self.assertFalse(self.patch_capture.exists()) + + def test_node_discovery_failure_prevents_kubernetes_credential_mutation(self) -> None: + """Fail closed before mutation when production nodes cannot be listed.""" + result = self._run_helper( + self._valid_config(), + FAKE_NODE_DISCOVERY_FAIL="true", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.talos_log.exists()) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.patch_capture.exists()) + + def test_ksail_lifecycle_wrapper_uses_only_sops_pull_token(self) -> None: + """Run create, reconcile, and update with the decrypted pull token.""" + self.assertTrue(KSAIL_PULL_WRAPPER.is_file()) + for command in ( + ("cluster", "create"), + ("workload", "reconcile"), + ("cluster", "update"), + ): + with self.subTest(command=command): + result = self._run_ksail_pull_wrapper(self._valid_config(), command) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + self.ksail_token_capture.read_text(encoding="utf-8"), + "fixture-secret-token", + ) + self.assertEqual( + self.ksail_username_capture.read_text(encoding="utf-8"), + "devantler", + ) + self.assertEqual( + self.ksail_command_capture.read_text(encoding="utf-8") + .strip() + .split(), + ["--config", "ksail.prod.yaml", *command], + ) + self.assertEqual( + self.ksail_config_path_capture.read_text(encoding="utf-8"), + "ksail.prod.yaml", + ) + self.assertEqual( + self.ksail_registry_capture.read_text(encoding="utf-8"), + "devantler:${GHCR_TOKEN}" + "@ghcr.io/devantler-tech/platform/manifests", + ) + self.assertEqual( + self.ksail_registry_override_capture.read_text( + encoding="utf-8" + ), + "${GHCR_USERNAME}:${GHCR_TOKEN}" + "@ghcr.io/devantler-tech/platform/manifests", + ) + self.assertRegex( + self.ksail_revision_capture.read_text(encoding="utf-8"), + r"^[0-9a-f]{64}$", + ) + self.assertNotIn( + "fixture-secret-token", + result.stdout + result.stderr, + ) + temporary_config = Path( + self.output_path_log.read_text(encoding="utf-8") + ) + self.assertFalse(temporary_config.exists()) + + def test_ksail_publish_wrapper_preserves_actions_write_token(self) -> None: + """Inject only the SOPS revision while preserving publication auth.""" + result = self._run_ksail_pull_wrapper( + self._valid_config(), + ("workload", "push"), + GITHUB_ACTOR="fixture-publisher", + GHCR_TOKEN="fixture-actions-write-token", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + self.ksail_token_capture.read_text(encoding="utf-8"), + "fixture-actions-write-token", + ) + self.assertEqual( + self.ksail_username_capture.read_text(encoding="utf-8"), + "fixture-publisher", + ) + self.assertEqual( + self.ksail_config_path_capture.read_text(encoding="utf-8"), + "ksail.prod.yaml", + ) + self.assertEqual( + self.ksail_registry_capture.read_text(encoding="utf-8"), + "devantler:${GHCR_TOKEN}@ghcr.io/devantler-tech/platform/manifests", + ) + self.assertEqual( + self.ksail_registry_override_capture.read_text(encoding="utf-8"), + "", + ) + self.assertRegex( + self.ksail_revision_capture.read_text(encoding="utf-8"), + r"^[0-9a-f]{64}$", + ) + self.assertNotIn( + "fixture-actions-write-token", + result.stdout + result.stderr, + ) + self.assertFalse( + self.output_path_log.exists(), + "publication needs the ciphertext revision but not decryption", + ) + + def test_production_config_keeps_its_protected_registry_template(self) -> None: + """Keep dynamic pull auth in the wrapper-owned environment only.""" + config = KSAIL_PROD_CONFIG.read_text(encoding="utf-8") + + self.assertIn( + 'registry: "devantler:${GHCR_TOKEN}' + '@ghcr.io/devantler-tech/platform/manifests"', + config, + ) + self.assertNotIn("${GHCR_USERNAME}:${GHCR_TOKEN}@ghcr.io", config) + + def test_lifecycle_preserves_username_from_sops_docker_config(self) -> None: + """Avoid hard-coding the pull credential's registry username.""" + config = self._valid_config() + config["auths"]["ghcr.io"]["username"] = "pull-robot" # type: ignore[index] + + result = self._run_ksail_pull_wrapper( + config, + ("cluster", "update"), + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + self.ksail_username_capture.read_text(encoding="utf-8"), + "pull-robot", + ) + + def test_ciphertext_rotation_changes_revision_without_hashing_token(self) -> None: + """Drive template drift from SOPS ciphertext rather than plaintext auth.""" + first = self._run_ksail_pull_wrapper( + self._valid_config(), + ("cluster", "update"), + ) + self.assertEqual(first.returncode, 0, first.stderr) + first_revision = self.ksail_revision_capture.read_text(encoding="utf-8") + + self._write_encrypted_secret("ENC[AES256_GCM,data:fixture-two]") + second = self._run_ksail_pull_wrapper( + self._valid_config(), + ("cluster", "update"), + ) + self.assertEqual(second.returncode, 0, second.stderr) + second_revision = self.ksail_revision_capture.read_text(encoding="utf-8") + + normalized_plaintext = json.dumps( + self._valid_config(), sort_keys=True, separators=(",", ":") + ) + plaintext_hash = hashlib.sha256( + f"{normalized_plaintext}\n".encode() + ).hexdigest() + self.assertNotEqual(first_revision, second_revision) + self.assertNotEqual(first_revision, plaintext_hash) + self.assertNotEqual(second_revision, plaintext_hash) + + def test_wrapper_rejects_arbitrary_commands(self) -> None: + """Never pass the decrypted pull credential to an arbitrary process.""" + result = self._run_ksail_pull_wrapper( + self._valid_config(), + ("workload", "delete"), + ) + + self.assertEqual(result.returncode, 64) + self.assertFalse(self.ksail_token_capture.exists()) + + def test_plaintext_revision_source_fails_closed(self) -> None: + """Refuse to hash a source value that is not SOPS ciphertext.""" + self._write_encrypted_secret("accidentally-plaintext") + + result = self._run_ksail_pull_wrapper( + self._valid_config(), + ("cluster", "update"), + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.ksail_token_capture.exists()) + + def test_accepts_standard_auth_only_docker_config(self) -> None: + """Accept Docker configs that store only the standard encoded auth field.""" + auth = base64.b64encode(b"devantler:fixture-secret-token").decode() + config = {"auths": {"ghcr.io": {"auth": auth}}} + + result = self._run_helper(config) + + self.assertEqual(result.returncode, 0, result.stderr) + patch = json.loads(self.patch_capture.read_text(encoding="utf-8")) + encoded = patch["data"][".dockerconfigjson"] + self.assertEqual(json.loads(base64.b64decode(encoded)), config) + + def test_accepts_matching_explicit_and_encoded_auth(self) -> None: + """Accept matching explicit and encoded GHCR credentials.""" + config = self._valid_config() + registry_auth = config["auths"]["ghcr.io"] + registry_auth["auth"] = base64.b64encode( + b"devantler:fixture-secret-token" + ).decode() + + result = self._run_helper(config) + + self.assertEqual(result.returncode, 0, result.stderr) + + def test_check_only_preflights_without_patching(self) -> None: + """Keep registry preflight mode free of Kubernetes mutations.""" + result = self._run_helper(self._valid_config(), ("--check-only",)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(self.kubectl_called.exists()) + self.assertFalse(self.patch_capture.exists()) + + def test_missing_or_malformed_registry_auth_fails_closed(self) -> None: + """Reject missing, malformed, empty, or contradictory GHCR credentials.""" + invalid_configs: list[object] = [ + {"auths": {"ghcr.io": {"username": "devantler"}}}, + {"auths": {"ghcr.io": {"username": "", "password": "token"}}}, + {"auths": {"ghcr.io": {"auth": "not-base64"}}}, + { + "auths": { + "ghcr.io": { + "username": "devantler", + "password": "fixture-secret-token", + "auth": base64.b64encode( + b"devantler:different-token" + ).decode(), + } + } + }, + {"auths": {}}, + "not-a-docker-config", + ] + + for config in invalid_configs: + with self.subTest(config=config): + result = self._run_helper(config) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.kubectl_called.exists()) + + def test_registry_denial_prevents_cluster_patch(self) -> None: + """Leave cluster credentials untouched when GHCR denies a manifest read.""" + result = self._run_helper( + self._valid_config(), + FAKE_CURL_DENY_REPOSITORY="devantler-tech/platform/manifests", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.kubectl_called.exists()) + + def test_token_success_without_registry_read_access_prevents_cluster_patch( + self, + ) -> None: + """Require package read access even when the token exchange succeeds.""" + result = self._run_helper( + self._valid_config(), + FAKE_CURL_DENY_REPOSITORY="devantler-tech/wedding-app", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.kubectl_called.exists()) + + def test_cluster_patch_failure_is_not_hidden(self) -> None: + """Propagate a failure to patch the root Flux pull Secret.""" + result = self._run_helper(self._valid_config(), FAKE_KUBECTL_FAIL="true") + + self.assertEqual(result.returncode, 43) + self.assertTrue(self.kubectl_called.exists()) + + def test_fresh_cluster_without_variables_base_skips_existing_fanout(self) -> None: + """Bootstrap only root auth before a fresh cluster creates its fan-out.""" + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_VARIABLES_BASE_ABSENT="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.patch_capture.exists()) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + + def test_missing_variables_base_fails_closed_without_bootstrap_mode(self) -> None: + """Require the explicit DR flag when the credential fan-out is absent.""" + result = self._run_helper( + self._valid_config(), + FAKE_VARIABLES_BASE_ABSENT="true", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.patch_capture.exists()) + + def test_partial_bootstrap_repairs_root_without_forcing_missing_fanout( + self, + ) -> None: + """Stage DR root auth without force-syncing an incomplete fan-out.""" + missing_resources = [ + "pushsecret/flux-system/seed-ghcr", + "externalsecret/wedding-app/ghcr-auth", + "externalsecret/ascoachingogvaner/ghcr-auth", + "externalsecret/kyverno/ghcr-auth", + ] + for resource in missing_resources: + with self.subTest(resource=resource): + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_MISSING_FANOUT_RESOURCE=resource, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.variables_patch_capture.exists()) + self.assertTrue(self.patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + self.assertIn("first reconcile will complete", result.stdout) + self.assertEqual( + self.operation_log.read_text(encoding="utf-8").splitlines()[-3:], + ["root-patch", "variables-patch", "root-patch"], + ) + + def test_partial_bootstrap_repairs_root_before_staging_variables(self) -> None: + """Keep an unavailable root patch from advancing partial consumers.""" + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_MISSING_FANOUT_RESOURCE="pushsecret/flux-system/seed-ghcr", + FAKE_KUBECTL_FAIL="true", + ) + + self.assertEqual(result.returncode, 43) + self.assertFalse(self.variables_patch_capture.exists()) + + def test_partial_fanout_fails_closed_without_bootstrap_mode(self) -> None: + """Reject a partial fan-out during a normal production deployment.""" + result = self._run_helper( + self._valid_config(), + FAKE_MISSING_FANOUT_RESOURCE="externalsecret/kyverno/ghcr-auth", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + + def test_missing_eso_crds_fails_without_staging_variables(self) -> None: + """Leave the fan-out source unchanged when normal mode is incomplete.""" + result = self._run_helper( + self._valid_config(), + FAKE_FANOUT_CRDS_ABSENT="true", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.variables_patch_capture.exists()) + self.assertFalse(self.patch_capture.exists()) + + def test_partial_bootstrap_without_eso_crds_repairs_root(self) -> None: + """Permit DR root repair before External Secrets CRDs exist.""" + result = self._run_helper( + self._valid_config(), + ("--allow-incomplete-fanout",), + FAKE_FANOUT_CRDS_ABSENT="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.variables_patch_capture.exists()) + self.assertTrue(self.patch_capture.exists()) + self.assertFalse(self.fanout_log.exists()) + + def test_pushsecret_sync_failure_is_not_hidden(self) -> None: + """Keep root auth unchanged when the OpenBao seed does not reconcile.""" + result = self._run_helper( + self._valid_config(), + FAKE_SYNC_STALL_RESOURCE="pushsecret/flux-system/seed-ghcr", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse( + self.patch_capture.exists(), + "root Flux auth must remain unchanged until fan-out verifies", + ) + self.assertTrue(self.variables_patch_capture.exists()) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + + def test_same_second_sync_accepts_controller_resource_version_edge(self) -> None: + """Accept a controller update when refreshTime has one-second precision.""" + result = self._run_helper( + self._valid_config(), + FAKE_SYNC_SAME_REFRESH_TIME="true", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(self.patch_capture.exists()) + + def test_materialised_consumer_mismatch_is_not_hidden(self) -> None: + """Reject fan-out completion when a workload Secret has stale content.""" + result = self._run_helper( + self._valid_config(), + FAKE_CONSUMER_MISMATCH_NAMESPACE="wedding-app", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "wedding-app/ghcr-auth did not materialise", + result.stdout + result.stderr, + ) + self.assertFalse( + self.patch_capture.exists(), + "root Flux auth must remain unchanged until consumers match", + ) + self.assertNotIn("fixture-secret-token", result.stdout + result.stderr) + + +class DeployActionOrderingTests(unittest.TestCase): + """Keep credential refreshes on both sides of KSail-owned mutations.""" + + def test_cluster_lifecycle_uses_sops_auth_but_publish_keeps_actions_token( + self, + ) -> None: + """Separate KSail pull lifecycle auth from the Actions publish token.""" + action = ACTION.read_text(encoding="utf-8") + workflow = DR_REBUILD.read_text(encoding="utf-8") + wrapper = "./scripts/run-ksail-prod-with-pull-auth.sh" + + action_reconcile = action.index("id: reconcile") + action_update = action.index("name: 🔄 Update cluster") + action_reassert = action.index("id: reassert_flux_ghcr_auth") + self.assertIn( + f"run: {wrapper} workload reconcile", + action[action_reconcile:action_update], + ) + self.assertNotIn("GHCR_TOKEN:", action[action_reconcile:action_update]) + self.assertIn( + f"run: {wrapper} cluster update", + action[action_update:action_reassert], + ) + self.assertNotIn("GHCR_TOKEN:", action[action_update:action_reassert]) + + action_push = action.index("name: 📦 Push manifests to GHCR") + action_sign = action.index("name: ⚙️ Install cosign") + self.assertIn( + f"run: {wrapper} workload push", + action[action_push:action_sign], + ) + self.assertIn( + "GHCR_TOKEN: ${{ inputs.ghcr-token }}", + action[action_push:action_sign], + ) + + dr_create = workflow.index("name: 🏗️ Create cluster") + dr_stage = workflow.index("id: stage_flux_ghcr_auth") + self.assertIn( + f"run: {wrapper} cluster create", + workflow[dr_create:dr_stage], + ) + self.assertNotIn("GHCR_TOKEN:", workflow[dr_create:dr_stage]) + + dr_push = workflow.index("name: 📦 Push manifests to GHCR") + dr_verify = workflow.index("id: verify_flux_ghcr_auth_after_push") + self.assertIn( + f"run: {wrapper} workload push", + workflow[dr_push:dr_verify], + ) + self.assertIn( + "GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}", + workflow[dr_push:dr_verify], + ) + + dr_reconcile = workflow.index("name: 🔁 Trigger Flux reconciliation") + dr_wait = workflow.index("name: ⏳ Wait for Flux to settle") + self.assertIn( + f"run: {wrapper} workload reconcile", + workflow[dr_reconcile:dr_wait], + ) + self.assertNotIn("GHCR_TOKEN:", workflow[dr_reconcile:dr_wait]) + + def test_talosctl_is_installed_before_any_mutating_bridge(self) -> None: + """Ensure both isolated Actions jobs can execute node synchronization.""" + action = ACTION.read_text(encoding="utf-8") + workflow = DR_REBUILD.read_text(encoding="utf-8") + + for document in (action, workflow): + with self.subTest(document=document[:40]): + setup = document.index("name: ⚙️ Setup talosctl") + stage = document.index("id: stage_flux_ghcr_auth") + self.assertLess(setup, stage) + setup_step = document[setup:stage] + self.assertIn("TALOS_VERSION: \"1.13.5\"", setup_step) + self.assertIn("talosctl-linux-amd64", setup_step) + if document == action: + restore = document.index("name: 🔑 Restore talosconfig") + self.assertLess(restore, stage) + + def test_consumer_staging_precedes_publish_and_is_reasserted_after_update( + self, + ) -> None: + """Keep full credential staging before publish and after cluster update.""" + action = ACTION.read_text(encoding="utf-8") + wrapper = "./scripts/run-ksail-prod-with-pull-auth.sh" + + first_refresh = action.index("id: stage_flux_ghcr_auth") + push = action.index(f"run: {wrapper} workload push") + post_push_refresh = action.index("id: verify_flux_ghcr_auth_after_push") + reconcile = action.index("id: reconcile") + cluster_update = action.index(f"run: {wrapper} cluster update") + final_refresh = action.index("id: reassert_flux_ghcr_auth\n") + + self.assertLess(first_refresh, push) + self.assertLess(push, post_push_refresh) + self.assertLess(post_push_refresh, reconcile) + self.assertLess(reconcile, cluster_update) + self.assertLess(cluster_update, final_refresh) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh\n", + action[first_refresh:push], + ) + self.assertNotIn( + "--check-only", + action[first_refresh:push], + ) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only", + action[post_push_refresh:reconcile], + ) + final_refresh_step = action[final_refresh:] + self.assertIn("always() &&", final_refresh_step) + self.assertIn( + "steps.verify_flux_ghcr_auth_after_push.outcome == 'success'", + final_refresh_step, + ) + self.assertIn( + "steps.reconcile.outcome == 'success'", + final_refresh_step, + ) + self.assertEqual(action.count("scripts/refresh-flux-ghcr-auth.sh"), 3) + + def test_disaster_rebuild_preflights_then_stages_before_publish( + self, + ) -> None: + """Keep DR credential checks around publish and OpenBao restoration.""" + workflow = DR_REBUILD.read_text(encoding="utf-8") + wrapper = "./scripts/run-ksail-prod-with-pull-auth.sh" + + preflight = workflow.index( + "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only" + ) + cluster_create = workflow.index(f"run: {wrapper} cluster create") + stage = workflow.index("id: stage_flux_ghcr_auth") + push = workflow.index(f"run: {wrapper} workload push") + verify = workflow.index("id: verify_flux_ghcr_auth_after_push") + fanout_verify = workflow.index("id: verify_flux_ghcr_fanout") + openbao_restore = workflow.index( + "name: 🔐 Restore OpenBao from the R2 snapshot mirror" + ) + post_restore_verify = workflow.index( + "id: reassert_flux_ghcr_after_restore" + ) + reconcile = workflow.index(f"run: {wrapper} workload reconcile") + + self.assertLess(preflight, cluster_create) + self.assertLess(cluster_create, stage) + self.assertLess(stage, push) + self.assertLess(push, verify) + self.assertLess(verify, reconcile) + self.assertLess(reconcile, fanout_verify) + self.assertLess(fanout_verify, openbao_restore) + self.assertLess(openbao_restore, post_restore_verify) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout", + workflow[stage:push], + ) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh --check-only", + workflow[verify:reconcile], + ) + self.assertIn( + "run: ./scripts/refresh-flux-ghcr-auth.sh\n", + workflow[fanout_verify:], + ) + self.assertIn( + "if: ${{ !cancelled() && inputs.restore && " + "steps.verify_flux_ghcr_fanout.outcome == 'success' }}", + workflow[post_restore_verify:], + ) + self.assertEqual(workflow.count("scripts/refresh-flux-ghcr-auth.sh"), 5) + + def test_manual_dr_waits_for_flux_before_full_bridge(self) -> None: + """Keep bootstrap mode until every first-reconcile layer is Ready.""" + runbook = DR_RUNBOOK.read_text(encoding="utf-8") + ci_workflow = CI_WORKFLOW.read_text(encoding="utf-8") + manual = runbook[ + runbook.index("# 2. Prove the Git/SOPS pull credential") : + runbook.index("# 6. ONLY if the OpenBao raft-snapshot") + ] + + bootstrap = manual.index( + "./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout" + ) + reconcile = manual.index( + "./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile" + ) + wait = manual.index( + "kubectl --context admin@prod -n flux-system wait" + ) + full_bridge = manual.index( + "./scripts/refresh-flux-ghcr-auth.sh # prove completed fan-out" + ) + + self.assertLess(bootstrap, reconcile) + self.assertLess(reconcile, wait) + self.assertLess(wait, full_bridge) + self.assertIn( + "workload reconciliation also requires the SOPS key", + runbook, + ) + self.assertIn("- 'docs/dr/runbook.md'", ci_workflow) + + def test_bridge_docs_and_tests_validate_without_deploying(self) -> None: + """Keep validation-only bridge changes out of production deploys.""" + workflow = CI_WORKFLOW.read_text(encoding="utf-8") + filters = workflow[ + workflow.index(" filters: |") : workflow.index("\n validate:") + ] + k8s_filter = filters[ + filters.index(" k8s:") : filters.index( + " bridge_validation:" + ) + ] + bridge_filter = filters[ + filters.index(" bridge_validation:") : filters.index( + " talos:" + ) + ] + + for validation_only_path in ( + "scripts/tests/test_refresh_flux_ghcr_auth.py", + "docs/dr/runbook.md", + ): + with self.subTest(path=validation_only_path): + quoted_path = f"- '{validation_only_path}'" + self.assertNotIn(quoted_path, k8s_filter) + self.assertIn(quoted_path, bridge_filter) + + self.assertIn( + "bridge_validation: ${{ steps.filter.outputs.bridge_validation }}", + workflow, + ) + validate = workflow[ + workflow.index(" validate:") : workflow.index(" naming:") + ] + self.assertIn( + "needs.changes.outputs.bridge_validation == 'true'", validate + ) + deploy = workflow[ + workflow.index(" deploy-prod:") : workflow.index( + " heal-prod-on-failure:" + ) + ] + heal = workflow[ + workflow.index(" heal-prod-on-failure:") : workflow.index( + " ci-required-checks:" + ) + ] + for production_job in (deploy, heal): + self.assertIn("needs.changes.outputs.k8s == 'true'", production_job) + self.assertNotIn("bridge_validation", production_job) + + +class ManualBridgePrerequisiteTests(unittest.TestCase): + """Keep manual recovery dependencies explicit and fail-fast.""" + + def test_yq_v4_is_documented_and_preflighted(self) -> None: + """Require the YAML tool used before any GHCR recovery mutation.""" + instructions = AGENT_INSTRUCTIONS.read_text(encoding="utf-8") + runbook = DR_RUNBOOK.read_text(encoding="utf-8") + library = GHCR_AUTH_LIB.read_text(encoding="utf-8") + + self.assertIn("yq v4", instructions) + self.assertIn("yq v4", runbook[: runbook.index("## Scenario 1")]) + self.assertIn("require_flux_ghcr_yaml_tool()", library) + for entrypoint in (HELPER, KSAIL_PULL_WRAPPER): + with self.subTest(entrypoint=entrypoint.name): + self.assertIn( + "require_flux_ghcr_yaml_tool", + entrypoint.read_text(encoding="utf-8"), + ) + + def test_yaml_tool_preflight_rejects_missing_or_incompatible_yq(self) -> None: + """Stop before entrypoint work when Mike Farah yq v4 is unavailable.""" + cases = ("missing", "incompatible") + for case in cases: + with self.subTest(case=case), tempfile.TemporaryDirectory() as temp_dir: + if case == "incompatible": + fake_yq = Path(temp_dir) / "yq" + fake_yq.write_text( + "#!/bin/bash\necho 'yq 3.4.3'\n", encoding="utf-8" + ) + fake_yq.chmod(0o755) + environment = os.environ.copy() + environment["PATH"] = temp_dir + + result = subprocess.run( + [ + "/bin/bash", + "-c", + 'source "$1"; require_flux_ghcr_yaml_tool', + "bash", + str(GHCR_AUTH_LIB), + ], + env=environment, + text=True, + capture_output=True, + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("yq v4 is required", result.stderr) + + +class RequiredPackageCoverageTests(unittest.TestCase): + """Keep pinned private provider references in the live GHCR preflight.""" + + def test_provider_upjet_unifi_reference_is_preflighted(self) -> None: + """Require the live private provider package in the GHCR preflight.""" + manifest = PROVIDER_UPJET_UNIFI.read_text(encoding="utf-8") + helper = HELPER.read_text(encoding="utf-8") + + package_line = next( + line.strip() + for line in manifest.splitlines() + if line.strip().startswith("package: ghcr.io/") + ) + package_reference = package_line.removeprefix("package: ghcr.io/") + + self.assertIn(f'"{package_reference}"', helper) + + def test_exact_declared_ksail_operator_image_is_preflighted(self) -> None: + """Bind the GHCR preflight to the chart's exact KSail image version.""" + helper = HELPER.read_text(encoding="utf-8") + + self.assertIn( + '"devantler-tech/ksail:v${KSAIL_OPERATOR_VERSION}"', helper + ) + self.assertIn(".spec.chart.spec.version", helper) + self.assertTrue(KSAIL_OPERATOR_VERSION) + + +class TalosRegistryAuthConfigTests(unittest.TestCase): + """Keep lifecycle drift and post-pull verification states distinct.""" + + def test_static_desired_revision_cannot_claim_verified_pull(self) -> None: + """Require helper proof before a new Talos node is considered current.""" + static_revision = TALOS_GHCR_REVISION.read_text(encoding="utf-8") + static_config = "\n".join( + line + for line in static_revision.splitlines() + if not line.lstrip().startswith("#") + ) + helper = HELPER.read_text(encoding="utf-8") + + self.assertIn("ghcr-pull-desired-revision", static_config) + self.assertNotIn("ghcr-pull-verified-revision", static_config) + self.assertIn("ghcr-pull-verified-revision", helper) + + def test_registry_auth_uses_supported_talos_document(self) -> None: + """Use one standalone RegistryAuthConfig document with variable auth.""" + registry_auth = TALOS_GHCR_AUTH.read_text(encoding="utf-8") + + self.assertIn("kind: RegistryAuthConfig", registry_auth) + self.assertIn("username: ${GHCR_USERNAME}", registry_auth) + self.assertIn("password: ${GHCR_TOKEN}", registry_auth) + self.assertNotIn("machine:\n", registry_auth) + self.assertNotIn("\n---\n", registry_auth) + + +if __name__ == "__main__": + unittest.main() diff --git a/talos/cluster/authenticate-ghcr-pulls.yaml b/talos/cluster/authenticate-ghcr-pulls.yaml index da40e8f0f..887b9e820 100644 --- a/talos/cluster/authenticate-ghcr-pulls.yaml +++ b/talos/cluster/authenticate-ghcr-pulls.yaml @@ -3,50 +3,42 @@ # talos/cluster/verify-first-party-images.yaml (ImageVerificationConfig) verifies the # cosign signature of every ghcr.io/devantler-tech/* image at the containerd # PULL layer. That signature is a separate artifact stored in the SAME package -# as the image, and Talos fetches it using THIS machine.registries config — +# as the image, and Talos fetches it using THIS RegistryAuthConfig — # NOT the per-pod `ghcr-auth` imagePullSecret (which kubelet only hands to # containerd for the image-layer pull). Our first-party packages are PRIVATE, -# so without a node-level ghcr.io credential the signature fetch goes out -# anonymous → HTTP 401 → verification fails → ImagePullBackOff. +# so without working node-level ghcr.io credentials the signature fetch is +# denied → verification fails → ImagePullBackOff. # # It's latent: an image already cached on a node is "already present" and never # re-verified, so it only bites a FRESH pull — a new tenant release, or an # existing image after a node roll evicts its cache (how ascoachingogvaner # v1.2.0 wedged on prod-worker-1 after the k8s-1.36 roll). # -# This is pure host AUTH (machine.registries.config), NOT a mirror -# (machine.registries.mirrors): it attaches a credential to ghcr.io pulls with +# This is pure host auth, not a mirror: it attaches a credential to ghcr.io pulls with # no endpoint redirect. (ksail's --mirror-registry flag is the wrong tool here — # it rewrites the endpoint to a local pull-through container that doesn't exist # on Hetzner, and it's only ever processed at `cluster create`, never on # `cluster update`.) It is the same credential the cluster already holds for the # Flux manifests artifact (spec.cluster.localRegistry), so it adds no new trust. # -# ${GHCR_TOKEN} is expanded by ksail when it loads this patch (envvar.ExpandBytes -# in LoadPatches) from the deploy step's env — the SAME env var localRegistry -# already requires — so nothing secret is committed. This is ksail's documented, -# supported config-as-code pattern for node registry auth (declarative-config -# docs, "Environment Variable Expansion"; ksail#5135) — explicitly NOT the -# --mirror-registry CLI flag. Prod-only by construction: this dir pairs with -# ksail.prod.yaml; local/CI uses talos-local/, which carries no -# ImageVerificationConfig. +# ${GHCR_TOKEN} is expanded by ksail when it loads this patch. Production +# cluster lifecycle commands inject the validated Git/SOPS pull token through +# run-ksail-prod-with-pull-auth.sh; workload push preserves the separate Actions +# write token. The separate mark-ghcr-pull-revision.yaml patch carries a +# non-secret SOPS ciphertext revision because KSail deliberately redacts +# registry credentials from config fingerprints; the marker makes token-only +# rotations update the cluster-autoscaler template used by future nodes. +# Nothing secret is committed. Prod-only by construction: local/CI uses +# talos-local/. # -# CAVEAT — applying to EXISTING nodes: ksail's `cluster update` diff only tracks -# node count / Talos version / ISO, so a machine-config-only change like this is -# NOT pushed to already-running nodes (it reports "No changes detected"). This -# patch makes every node correct-by-construction on create/recreate (and the -# autoscaler's rendered worker config); to apply it to nodes that are already up, -# patch them once out-of-band, e.g.: -# talosctl --nodes patch mc --patch @ -# (registries is a no-reboot field), then restart any pod stuck in -# ImagePullBackOff. See docs/dr/runbook.md. +# Existing nodes are synchronized separately by refresh-flux-ghcr-auth.sh using +# this same RegistryAuthConfig shape, one node at a time in no-reboot mode, and +# each node must then pull the exact live ksail-operator image successfully. # # Reference: -# https://docs.siderolabs.com/talos/v1.13/reference/configuration/network/registriesconfig/ -machine: - registries: - config: - ghcr.io: - auth: - username: devantler - password: ${GHCR_TOKEN} +# https://docs.siderolabs.com/talos/v1.13/reference/configuration/cri/registryauthconfig +apiVersion: v1alpha1 +kind: RegistryAuthConfig +name: ghcr.io +username: ${GHCR_USERNAME} +password: ${GHCR_TOKEN} diff --git a/talos/cluster/mark-ghcr-pull-revision.yaml b/talos/cluster/mark-ghcr-pull-revision.yaml new file mode 100644 index 000000000..49f24bbd8 --- /dev/null +++ b/talos/cluster/mark-ghcr-pull-revision.yaml @@ -0,0 +1,14 @@ +# Non-secret rotation marker for the node-level GHCR pull credential. +# +# KSail redacts RegistryAuthConfig credentials when it fingerprints Talos +# machine configuration and autoscaler templates. Hashing the committed SOPS +# ciphertext gives token rotations a visible, non-secret input without +# publishing a stable hash of the decrypted token. Production lifecycle +# commands provide ${GHCR_PULL_REVISION}. This desired marker intentionally +# differs from the helper-owned ghcr-pull-verified-revision annotation: a newly +# created/autoscaled node must not claim a successful pull before it has proved +# one. The helper records that second marker only after an exact private KSail +# image pull succeeds. +machine: + nodeAnnotations: + platform.devantler.tech/ghcr-pull-desired-revision: ${GHCR_PULL_REVISION}