fix(deploy): synchronize production GHCR pull auth

.github/actions/deploy-prod/action.yml

@@ -19,7 +19,7 @@ inputs:
     description: talosconfig for the prod cluster (secrets.TALOS_CONFIG) used by `ksail cluster update`.
     required: true
   ghcr-token:
-    description: GHCR PAT with write:packages (secrets.GHCR_TOKEN) for OCI push/pull and cosign auth.
+    description: GHCR PAT with write:packages (secrets.GHCR_TOKEN) for OCI push and signing; every pull path uses the Git/SOPS credential.
     required: true
   hcloud-token:
     description: Hetzner Cloud API token (secrets.HCLOUD_TOKEN) used by the KSail Hetzner provider and CCM/CSI.
@@ -50,6 +50,19 @@ runs:
         sudo install /tmp/ksail /usr/local/bin/ksail
         ksail --version
 
+    - name: ⚙️ Setup talosctl
+      # refresh-flux-ghcr-auth.sh uses the host API for a secret-safe,
+      # per-node RegistryAuthConfig patch and exact image-pull proof.
+      shell: bash
+      env:
+        # Keep in step with spec.cluster.talos.version in ksail.prod.yaml.
+        # renovate: datasource=github-releases depName=siderolabs/talos extractVersion=^v(?<version>.+)$
+        TALOS_VERSION: "1.13.5"
+      run: |
+        curl -fsSL "https://github.com/siderolabs/talos/releases/download/v${TALOS_VERSION}/talosctl-linux-amd64" -o /tmp/talosctl
+        sudo install /tmp/talosctl /usr/local/bin/talosctl
+        talosctl version --client
+
     - name: 🔐 Create SOPS Age key
       shell: bash
       env:
@@ -126,6 +139,15 @@ runs:
         echo "::error::Prod cluster (context admin@prod) is unreachable with the restored kubeconfig (last kubectl error: ${last_err}). KUBE_CONFIG (and likely TALOS_CONFIG) are probably stale after a cluster rebuild — refresh both and update the 'prod' environment secrets. See docs/dr/runbook.md → 'Refresh CI deploy credentials'."
         exit 1
 
+    - name: 🔑 Stage Flux and consumer GHCR pull credential
+      id: stage_flux_ghcr_auth
+      # Validate all seven packages, synchronize and prove the exact live image
+      # pull on every stale Talos node, then stage and verify the Kubernetes fan-out
+      # BEFORE publishing latest. Every pull consumer has one known-readable
+      # credential before the mutable tag can be observed.
+      shell: bash
+      run: ./scripts/refresh-flux-ghcr-auth.sh
+
     # Publish the GitOps manifests (push → sign → attest → reconcile) BEFORE
     # syncing Talos machine config (the "🔄 Update cluster" step runs LAST).
     # Manifest delivery never touches the nodes, so it must not be gated on
@@ -139,7 +161,7 @@ runs:
         GITHUB_ACTOR: ${{ github.actor }}
         GHCR_TOKEN: ${{ inputs.ghcr-token }}
         HCLOUD_TOKEN: ${{ inputs.hcloud-token }}
-      run: ksail --config ksail.prod.yaml workload push
+      run: ./scripts/run-ksail-prod-with-pull-auth.sh workload push
 
     - name: ⚙️ Install cosign
       # sigstore/cosign-installer is the official installer action — it
@@ -284,14 +306,20 @@ runs:
         push-to-registry: true
         create-storage-record: false
 
+    - name: 🔎 Verify Flux GHCR pull credential after publish
+      id: verify_flux_ghcr_auth_after_push
+      # Revalidate the newly-published artifact before apps reconcile. The
+      # complete consumer fan-out was staged before the mutable tag moved.
+      shell: bash
+      run: ./scripts/refresh-flux-ghcr-auth.sh --check-only
+
     - name: 🔁 Trigger Flux reconciliation
       id: reconcile
       shell: bash
       env:
         GITHUB_ACTOR: ${{ github.actor }}
-        GHCR_TOKEN: ${{ inputs.ghcr-token }}
         HCLOUD_TOKEN: ${{ inputs.hcloud-token }}
-      run: ksail --config ksail.prod.yaml workload reconcile
+      run: ./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile
 
     - name: 🩺 Diagnose Flux on failure
       if: failure() && steps.reconcile.outcome == 'failure'
@@ -314,10 +342,24 @@ runs:
       # release).
       shell: bash
       env:
-        GHCR_TOKEN: ${{ inputs.ghcr-token }}
         HCLOUD_TOKEN: ${{ inputs.hcloud-token }}
         # ksail env-expands ${WG_SERVER_PRIVATE_KEY} in the talos patches at
         # load time; an unset var renders an empty key and Talos rejects the
         # whole machine config ('private key is invalid: wrong key "" length: 0').
         WG_SERVER_PRIVATE_KEY: ${{ inputs.wg-server-private-key }}
-      run: ksail --config ksail.prod.yaml cluster update
+      run: ./scripts/run-ksail-prod-with-pull-auth.sh cluster update
+
+    - name: 🔑 Reassert Flux and consumer GHCR pull credential
+      id: reassert_flux_ghcr_auth
+      # cluster update is already wrapped with Git/SOPS auth, but reassert the
+      # full transaction afterward in case KSail fails part-way. Reassert after
+      # its synchronous consumer fan-out when that update fails part-way, but
+      # only after the publish, signing, attestation, post-publish auth check,
+      # and Flux reconcile all succeeded. Never mutate auth as cleanup for a
+      # failed delivery.
+      if: >-
+        always() &&
+        steps.verify_flux_ghcr_auth_after_push.outcome == 'success' &&
+        steps.reconcile.outcome == 'success'
+      shell: bash
+      run: ./scripts/refresh-flux-ghcr-auth.sh

.github/workflows/ci.yaml

@@ -19,6 +19,7 @@ jobs:
       pull-requests: read # detect changed files
     outputs:
       k8s: ${{ steps.filter.outputs.k8s }}
+      bridge_validation: ${{ steps.filter.outputs.bridge_validation }}
       talos: ${{ steps.filter.outputs.talos }}
     steps:
       - name: 📑 Checkout
@@ -41,6 +42,20 @@ jobs:
               - 'talos/**'
               - 'scripts/validate-naming.py'
               - 'scripts/validate-embedded-json.py'
+              - 'scripts/refresh-flux-ghcr-auth.sh'
+              - 'scripts/ghcr-auth-lib.sh'
+              - 'scripts/run-ksail-prod-with-pull-auth.sh'
+              - '.github/actions/deploy-prod/**'
+              - '.github/workflows/dr-rebuild.yaml'
+              - '.github/workflows/ci.yaml'
+            bridge_validation:
+              - 'scripts/refresh-flux-ghcr-auth.sh'
+              - 'scripts/ghcr-auth-lib.sh'
+              - 'scripts/run-ksail-prod-with-pull-auth.sh'
+              - 'scripts/tests/test_refresh_flux_ghcr_auth.py'
+              - 'docs/dr/runbook.md'
+              - '.github/actions/deploy-prod/**'
+              - '.github/workflows/dr-rebuild.yaml'
               - '.github/workflows/ci.yaml'
             talos:
               - 'talos/**'
@@ -55,7 +70,10 @@ jobs:
     # longer boots it; instead it schema-validates and security-scans the
     # manifests. This needs no secrets, so — unlike the old system test — it also
     # runs on fork PRs.
-    if: github.event_name == 'pull_request' && needs.changes.outputs.k8s == 'true'
+    if: >-
+      github.event_name == 'pull_request' &&
+      (needs.changes.outputs.k8s == 'true' ||
+      needs.changes.outputs.bridge_validation == 'true')
     runs-on: ubuntu-latest
     permissions:
       contents: read # checkout repository
@@ -65,6 +83,20 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: 🔐 Validate Flux GHCR credential bridge
+        # Pure local tests with fake ksail/docker/kubectl commands: no SOPS key,
+        # registry credential, cluster access, or network is used.
+        run: |
+          bash -n \
+            scripts/ghcr-auth-lib.sh \
+            scripts/refresh-flux-ghcr-auth.sh \
+            scripts/run-ksail-prod-with-pull-auth.sh
+          shellcheck \
+            scripts/ghcr-auth-lib.sh \
+            scripts/refresh-flux-ghcr-auth.sh \
+            scripts/run-ksail-prod-with-pull-auth.sh
+          python3 -m unittest scripts.tests.test_refresh_flux_ghcr_auth
+
       - name: 🧩 Validate embedded JSON blobs
         # ConfigMaps that embed whole JSON documents as block scalars (e.g. the
         # Headlamp Kubescape exceptions ConfigMap's exceptionPolicies) are opaque
@@ -230,12 +262,16 @@ jobs:
 
           # Placeholder key material for env-expanded patch values (ksail
           # expands these from environment secrets at deploy time — see
-          # talos/control-planes/wireguard.yaml and talos/cluster/registry-auth.yaml).
+          # talos/control-planes/wireguard.yaml and
+          # talos/cluster/authenticate-ghcr-pulls.yaml and
+          # talos/cluster/mark-ghcr-pull-revision.yaml).
           # Real-format but throwaway, so validation exercises the expansion
           # path without any real secret on the PR event.
           WG_SERVER_PRIVATE_KEY="$(openssl rand -base64 32)"
+          GHCR_USERNAME="placeholder-ghcr-user"
           GHCR_TOKEN="placeholder-ghcr-token"
-          export WG_SERVER_PRIVATE_KEY GHCR_TOKEN
+          GHCR_PULL_REVISION="$(printf '%s\n' 'ENC[ci-placeholder-ciphertext]' | shasum -a 256 | awk '{print $1}')"
+          export WG_SERVER_PRIVATE_KEY GHCR_USERNAME GHCR_TOKEN GHCR_PULL_REVISION
 
           validate_overlay() {
             local dir="$1" mode="$2"

.github/workflows/dr-rebuild.yaml

@@ -85,6 +85,17 @@ jobs:
           sudo install /tmp/ksail /usr/local/bin/ksail
           ksail --version
 
+      - name: ⚙️ Setup talosctl
+        # The GHCR bridge verifies RegistryAuthConfig on each stale node.
+        env:
+          # Keep in step with spec.cluster.talos.version in ksail.prod.yaml.
+          # renovate: datasource=github-releases depName=siderolabs/talos extractVersion=^v(?<version>.+)$
+          TALOS_VERSION: "1.13.5"
+        run: |
+          curl -fsSL "https://github.com/siderolabs/talos/releases/download/v${TALOS_VERSION}/talosctl-linux-amd64" -o /tmp/talosctl
+          sudo install /tmp/talosctl /usr/local/bin/talosctl
+          talosctl version --client
+
       - name: 🔐 Create SOPS Age key
         env:
           SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
@@ -94,33 +105,51 @@ jobs:
           echo "${SOPS_AGE_KEY}" > ~/.config/sops/age/keys.txt
           chmod 600 ~/.config/sops/age/keys.txt
 
+      - name: 🔎 Preflight Flux GHCR pull credential
+        # Validate every package served by the shared Git/SOPS credential
+        # before creating infrastructure or publishing a mutable latest tag.
+        run: ./scripts/refresh-flux-ghcr-auth.sh --check-only
+
       - name: 🏗️ Create cluster
         # From-zero provisioning: Hetzner servers, Talos boot, CCM, CSI. Writes
         # a fresh kubeconfig (~/.kube/config, context admin@prod) and
         # talosconfig (~/.talos/config) on this runner — every later step uses
         # those, so this workflow does NOT depend on the (now stale)
         # KUBE_CONFIG / TALOS_CONFIG environment secrets.
-        run: ksail --config ksail.prod.yaml cluster create
+        run: ./scripts/run-ksail-prod-with-pull-auth.sh cluster create
         env:
-          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
           # ksail env-expands ${WG_SERVER_PRIVATE_KEY} in the talos patches at
           # load time (talos/control-planes/wireguard.yaml); an unset var
           # renders an empty key and Talos rejects the whole machine config.
           WG_SERVER_PRIVATE_KEY: ${{ secrets.WG_SERVER_PRIVATE_KEY }}
 
+      - name: 🔑 Stage Flux and consumer GHCR pull credential
+        id: stage_flux_ghcr_auth
+        # cluster create already uses the Git/SOPS pull token. Reassert it on
+        # every current Talos node and make Git/SOPS authoritative before the
+        # mutable tag is published. A
+        # partial bootstrap may have variables-base but no ESO resources yet;
+        # the bridge safely repairs root auth so Flux can create them.
+        run: ./scripts/refresh-flux-ghcr-auth.sh --allow-incomplete-fanout
+
       - name: 📦 Push manifests to GHCR
-        run: ksail --config ksail.prod.yaml workload push
+        run: ./scripts/run-ksail-prod-with-pull-auth.sh workload push
         env:
           GITHUB_ACTOR: ${{ github.actor }}
           GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
 
+      - name: 🔎 Verify Flux GHCR pull credential after publish
+        id: verify_flux_ghcr_auth_after_push
+        # Prove the newly-published artifact is readable before reconciliation;
+        # every available consumer was already staged before push.
+        run: ./scripts/refresh-flux-ghcr-auth.sh --check-only
+
       - name: 🔁 Trigger Flux reconciliation
-        run: ksail --config ksail.prod.yaml workload reconcile
+        run: ./scripts/run-ksail-prod-with-pull-auth.sh workload reconcile
         env:
           GITHUB_ACTOR: ${{ github.actor }}
-          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
 
       - name: ⏳ Wait for Flux to settle
@@ -132,6 +161,13 @@ jobs:
           done
           echo "✅ All Flux Kustomizations Ready — fresh platform converged."
 
+      - name: 🔑 Verify completed GHCR credential fan-out
+        id: verify_flux_ghcr_fanout
+        # Bootstrap mode may repair only root auth when ESO is not installed
+        # yet. After Flux converges, require the complete PushSecret /
+        # ExternalSecret transaction and exact consumer payload verification.
+        run: ./scripts/refresh-flux-ghcr-auth.sh
+
       - name: 💾 Velero resource restore (newest Completed backup)
         if: ${{ inputs.restore }}
         run: |
@@ -372,6 +408,16 @@ jobs:
           echo "::endgroup::"
           echo "✅ OpenBao restored from the raft snapshot mirror."
 
+      - name: 🔑 Reassert GHCR credential after OpenBao restore
+        id: reassert_flux_ghcr_after_restore
+        if: ${{ !cancelled() && inputs.restore && steps.verify_flux_ghcr_fanout.outcome == 'success' }}
+        # The raft snapshot may contain an older infrastructure/ghcr/auth value.
+        # !cancelled() is intentional: even a failed restore can already have
+        # rematerialised that stale value, so attempt the fail-closed repair.
+        # Re-stage Git/SOPS in PushSecret-first order, wait for every consumer,
+        # compare the materialized payloads, and only then reassert root auth.
+        run: ./scripts/refresh-flux-ghcr-auth.sh
+
       - name: 🔑 Refresh CI deploy credentials (KUBE_CONFIG / TALOS_CONFIG)
         env:
           DR_GH_ADMIN_TOKEN: ${{ secrets.DR_GH_ADMIN_TOKEN }}