fix/backups

ansible/playbooks/backup-cloud.yml

@@ -111,7 +111,21 @@
           set -euo pipefail
           export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
 
-          TMPDIR=$(mktemp -d -t backup-cloud-XXXXXX)
+          # STABLE staging path — deliberately NOT mktemp.
+          #
+          # This directory is passed to `restic backup` as a source, so its name
+          # becomes part of the snapshot's path list. With mktemp the name was
+          # different every run (/tmp/backup-cloud-PX4ZQS, -pOvw3A, ...), and
+          # since `restic forget` groups by host+paths by default, every snapshot
+          # formed its own retention group. --keep-daily 7 --keep-weekly 4
+          # --keep-monthly 3 then dutifully kept the single member of each group,
+          # so R2 retention deleted nothing, ever, on a 10 GB free tier.
+          # A fixed path also means every snapshot has the same tree layout, so
+          # restore procedures can hardcode paths. See docs/BACKUP-RESTORE.md.
+          TMPDIR=/var/tmp/backup-cloud-stage
+          rm -rf "$TMPDIR"
+          mkdir -p "$TMPDIR"
+          chmod 700 "$TMPDIR"
           trap "rm -rf $TMPDIR" EXIT
 
           log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; }
@@ -146,6 +160,24 @@
             --no-verify-ssl \
             --quiet
 
+          # The k3s server token is REQUIRED to restore any etcd snapshot onto new
+          # hardware: k3s derives an AES-256 key from it (PBKDF2) to encrypt
+          # confidential data — CA private keys, bootstrap data — inside the
+          # datastore itself. Without it the snapshots in this very backup are
+          # unusable. It was previously backed up nowhere at all; it existed only on
+          # the three server nodes' local disks, so a total-site loss meant every
+          # etcd snapshot was decorative.
+          #
+          # SECURITY TRADEOFF, read before changing: snapshot + token together is
+          # full cluster compromise (see docs.k3s.io/cli/etcd-snapshot#security).
+          # They are co-located here only because this restic repo is encrypted as a
+          # whole, and its password lives in Vault and on the H4 — not in R2. Anyone
+          # able to decrypt this repo already holds the keys to everything in it.
+          # The offline break-glass envelope described in docs/BACKUP-RESTORE.md is
+          # still the primary recovery path; this is the convenience copy.
+          log "Copying k3s server token (required to restore etcd snapshots)..."
+          install -m 0600 /var/lib/rancher/k3s/server/token "$TMPDIR/k3s-server-token"
+
           # Build source list — include vault-snapshots only if the directory exists
           SOURCES=(
             /mnt/cold-8t/k3s-etcd-snapshots
@@ -157,7 +189,11 @@
           restic backup "${SOURCES[@]}" --tag cloud --host h4-core
 
           log "Pruning old cloud snapshots..."
-          restic forget --prune \
+          # --group-by host,tags (not the default host,paths): all cloud snapshots
+          # share host+tag, so they form ONE retention series. Belt-and-braces
+          # alongside the stable staging path above — if a future source path ever
+          # varies again, retention still works.
+          restic forget --prune --group-by host,tags \
             --keep-daily 7 --keep-weekly 4 --keep-monthly 3
 
           log "Cloud backup complete."

ansible/playbooks/backup.yml

@@ -1,18 +1,41 @@
 ---
 # Two backup streams onto the cold tier (8TB primary + 6TB copy), via host systemd timers:
-#   - daily : restic snapshot of live NAS data  (online, no downtime)
-#   - weekly: MicroShift data/etcd snapshot      (brief downtime in a 03:00 window)
+#   - daily 01:30: restic snapshot of live NAS data          (online, no downtime)
+#   - daily 00:30: copy of k3s' own etcd snapshots to cold   (online, no downtime)
 #
 # Cluster *config* is already safe in git (Argo); these cover *data* and *state*.
+#
+# Note on the etcd stream: k3s takes embedded-etcd snapshots itself and keeps them
+# on the NVMe. We do not snapshot — we replicate k3s' output onto the cold mirror
+# and verify it. See ansible/templates/backup-etcd.sh.j2 for the full history of
+# why this used to be a sqlite3 call and why it silently produced nothing.
 - name: Configure backups to the cold tier
   hosts: microshift
   become: true
   vars:
     restic_repo: "{{ cold_primary_mount }}/restic"          # 8TB primary
     restic_password_file: /etc/restic/password   # vaulted; created out of band
     restic_copy_repo: "{{ cold_secondary_mount }}/restic"   # 6TB redundant copy
-    etcd_backup_dir: "{{ cold_primary_mount }}/microshift-backups"
     restic_offsite_repo: "{{ offsite_restic_repo | default('') }}"   # e.g. b2:bucket:lab — set via host/group vars or Vault; creds in /etc/restic/offsite.env
+
+    # --- k3s etcd snapshot replication ---
+    k3s_snapshot_dir: /var/lib/rancher/k3s/server/db/snapshots  # k3s writes here (NVMe)
+    etcd_snapshot_dest: "{{ cold_primary_mount }}/k3s-etcd-snapshots"
+    etcd_snapshot_keep: 30            # ~15 days at k3s' 12h cadence; ~1 GB at 32 MB each
+    etcd_snapshot_min_bytes: 1048576  # 1 MiB — a real snapshot is ~30 MB; 4 KB was the bug
+    etcd_snapshot_max_age_seconds: 90000  # 25h, matching the LabBackupEtcdSilent alert
+
+    # --- Retention for the cold-sec copy (M-new-1) ---
+    # Deliberately DEEPER than the primary's 7/4/6: cold-sec is the long-history
+    # archive, not a mirror. Until 2026-07-25 it was never pruned at all.
+    copy_keep_daily: 14
+    copy_keep_weekly: 8
+    copy_keep_monthly: 12
+
+    # --- Weekly verification ---
+    cloud_env_file: /etc/restic/cloud.env   # written by backup-cloud.yml
+    cloud_stage_dir: /var/tmp/backup-cloud-stage  # must match backup-cloud.yml's TMPDIR
+    verify_max_snapshot_age_seconds: 172800 # 48h — weekly check, daily backups
   tasks:
     - name: Install restic
       ansible.builtin.package:
@@ -26,6 +49,14 @@
         mode: "0700"
 
     # --- NAS data backup (restic, daily) ---
+    - name: Render restic copy-to-secondary script
+      ansible.builtin.template:
+        src: ../templates/backup-nas-copy.sh.j2
+        dest: /usr/local/sbin/backup-nas-copy.sh
+        owner: root
+        group: root
+        mode: "0750"
+
     - name: Render restic backup service + timer
       ansible.builtin.template:
         src: "../templates/{{ item }}.j2"
@@ -35,8 +66,16 @@
         - backup-nas.service
         - backup-nas.timer
 
-    # --- MicroShift state backup (weekly, maintenance window) ---
-    - name: Render etcd/data backup service + timer
+    # --- k3s etcd snapshot replication (daily) ---
+    - name: Render etcd snapshot copy script
+      ansible.builtin.template:
+        src: ../templates/backup-etcd.sh.j2
+        dest: /usr/local/sbin/backup-etcd.sh
+        owner: root
+        group: root
+        mode: "0750"
+
+    - name: Render etcd backup service + timer
       ansible.builtin.template:
         src: "../templates/{{ item }}.j2"
         dest: "/etc/systemd/system/{{ item }}"
@@ -55,6 +94,24 @@
         - backup-offsite.service
         - backup-offsite.timer
 
+    # --- Weekly verification that the backups are real (not merely running) ---
+    - name: Render backup verification script
+      ansible.builtin.template:
+        src: ../templates/backup-verify.sh.j2
+        dest: /usr/local/sbin/backup-verify.sh
+        owner: root
+        group: root
+        mode: "0750"
+
+    - name: Render backup verification service + timer
+      ansible.builtin.template:
+        src: "../templates/{{ item }}.j2"
+        dest: "/etc/systemd/system/{{ item }}"
+        mode: "0644"
+      loop:
+        - backup-verify.service
+        - backup-verify.timer
+
     - name: Reload systemd
       ansible.builtin.systemd:
         daemon_reload: true
@@ -68,3 +125,4 @@
         - backup-nas.timer
         - backup-etcd.timer
         - backup-offsite.timer
+        - backup-verify.timer

ansible/playbooks/healthchecks.yml

@@ -9,7 +9,8 @@
 #   2. Create checks with the recommended settings below:
 #        backup-cloud   Period 25h,  Grace 1h   (daily 03:00 UTC on h4-core)
 #        backup-nas     Period 25h,  Grace 1h   (daily 01:30 UTC on h4-core)
-#        backup-etcd    Period 8d,   Grace 4h   (weekly on h4-core)
+#        backup-etcd    Period 25h,  Grace 1h   (daily 00:30 UTC on h4-core)
+#        backup-verify  Period 8d,   Grace 12h  (weekly Sun 04:00 UTC on h4-core)
 #        backup-vault   Period 25h,  Grace 1h   (daily 02:30 UTC on rpi5)
 #        h4-core        Period 5min, Grace 5min (host-alive heartbeat)
 #   3. Store all ping URLs in Vault:
@@ -40,6 +41,7 @@
       - backup-cloud
       - backup-nas
       - backup-etcd
+      - backup-verify
       - h4-core
 
   tasks:
@@ -74,6 +76,11 @@
         owner: root
         group: root
       loop: "{{ h4_checks }}"
+      # Skip checks that have no URL in Vault yet, so adding a new check to
+      # h4_checks does not break the play before the operator has created the
+      # healthchecks.io check and run `vault kv patch secret/lab/healthchecks`.
+      # hc-ping already no-ops safely when a .url file is absent.
+      when: item in hc
       no_log: true
 
     - name: Install hc-ping helper
@@ -110,6 +117,7 @@
         - backup-cloud
         - backup-nas
         - backup-etcd
+        - backup-verify
 
     - name: Install healthchecks drop-ins for H4 backup services
       ansible.builtin.copy:
@@ -124,6 +132,7 @@
         - backup-cloud
         - backup-nas
         - backup-etcd
+        - backup-verify
       notify: reload systemd daemon
 
     - name: Install h4-heartbeat systemd service

ansible/templates/backup-etcd.service.j2

@@ -1,16 +1,12 @@
 [Unit]
-Description=k3s state backup to cold tier (8TB)
+Description=Copy k3s etcd snapshots to cold tier (8TB)
 RequiresMountsFor={{ cold_primary_mount }}
 
 [Service]
 Type=oneshot
-# k3s single-node uses SQLite (state.db) — no embedded etcd.
-# sqlite3 .backup does an online hot-copy (safe while k3s is running).
-# Keeps the last 7 copies; prunes older ones.
-ExecStart=/bin/bash -c '\
-  DB=/var/lib/rancher/k3s/server/db/state.db && \
-  DEST={{ cold_primary_mount }}/k3s-etcd-snapshots && \
-  SNAP="$DEST/state-$(date +%%F_%%H%%M).db" && \
-  mkdir -p "$DEST" && \
-  sqlite3 "$DB" ".backup $SNAP" && \
-  ls -1t "$DEST"/state-*.db | tail -n +8 | xargs -r rm -f'
+# k3s owns snapshotting itself (embedded etcd, its own schedule and retention,
+# written to the NVMe). This unit only replicates those snapshots onto the cold
+# mirror and verifies what landed. The logic lives in a script rather than an
+# inline ExecStart so it can fail with a real exit code — see the header of
+# backup-etcd.sh for why that matters here.
+ExecStart=/usr/local/sbin/backup-etcd.sh

ansible/templates/backup-etcd.sh.j2

@@ -0,0 +1,108 @@
+#!/bin/bash
+# {{ ansible_managed | default('Managed by Ansible — do not edit by hand') }}
+#
+# Replicate k3s embedded-etcd snapshots onto the cold tier.
+#
+# k3s takes its own etcd snapshots on a schedule (default every 12h, retention 5)
+# into ${SRC} on the NVMe. That is the *same failure domain* as the cluster they
+# protect, so this script's only job is to copy them onto the cold mirror. It does
+# NOT take snapshots itself — k3s owns that.
+#
+# History (2026-07-25): this unit previously ran
+#   sqlite3 /var/lib/rancher/k3s/server/db/state.db ".backup ..."
+# which was correct only while k3s used the SQLite datastore. The 2026-07-02 HA
+# migration switched the H4 to embedded etcd and renamed state.db to
+# state.db.migrated. sqlite3 then silently recreated an empty state.db and emitted
+# 4 KB "backups" every week while still exiting 0 — so the unit reported success,
+# healthchecks.io stayed green, and backup-cloud shipped the empty files offsite.
+# Every failure path below therefore exits non-zero on purpose: a backup that
+# cannot prove it worked must fail loudly.
+#
+# NOTE FOR EDITORS: this file is a Jinja2 template. Do not use bash's array-length
+# syntax (dollar-brace-hash-name) here — brace followed by hash opens a Jinja
+# comment and the template fails to render with "Missing end of comment tag".
+# That is why the retention logic below uses a mktemp listing rather than an array.
+
+set -euo pipefail
+
+SRC={{ k3s_snapshot_dir | quote }}
+DEST={{ etcd_snapshot_dest | quote }}
+KEEP={{ etcd_snapshot_keep }}
+MIN_BYTES={{ etcd_snapshot_min_bytes }}
+MAX_AGE={{ etcd_snapshot_max_age_seconds }}
+
+if [ ! -d "$SRC" ]; then
+  echo "FAIL: k3s snapshot dir $SRC does not exist." >&2
+  echo "      Is this host still an etcd server node? Check:" >&2
+  echo "        kubectl get node \$(hostname) -o jsonpath='{.metadata.labels}'" >&2
+  exit 1
+fi
+
+mkdir -p "$DEST"
+
+# ---- Copy anything not already on cold ------------------------------------
+# k3s retains only ~5 snapshots (2.5 days at the 12h default), so a daily run
+# that copies every not-yet-present file captures the full series without gaps.
+shopt -s nullglob
+copied=0
+seen=0
+for src_file in "$SRC"/etcd-snapshot-*; do
+  seen=$((seen + 1))
+  base=$(basename "$src_file")
+  [ -e "$DEST/$base" ] && continue
+  # Stage then rename, so an interrupted run never publishes a truncated file.
+  cp -p -- "$src_file" "$DEST/.${base}.partial"
+  mv -- "$DEST/.${base}.partial" "$DEST/$base"
+  copied=$((copied + 1))
+  echo "copied $base"
+done
+
+if [ "$seen" -eq 0 ]; then
+  echo "FAIL: no etcd-snapshot-* files in $SRC — k3s is not producing snapshots." >&2
+  echo "      Check: journalctl -u k3s | grep -i snapshot" >&2
+  exit 1
+fi
+
+# ---- Verify what actually landed ------------------------------------------
+# The old unit's fatal flaw was trusting exit 0. Prove freshness and size instead.
+listing=$(mktemp)
+trap 'rm -f -- "$listing"' EXIT
+ls -1t -- "$DEST"/etcd-snapshot-* > "$listing" 2>/dev/null || true
+
+count=$(wc -l < "$listing" | tr -d '[:space:]')
+newest=$(head -n 1 "$listing")
+
+if [ -z "$newest" ]; then
+  echo "FAIL: no snapshots present in $DEST after copy." >&2
+  exit 1
+fi
+
+size=$(stat -c %s -- "$newest")
+age=$(( $(date +%s) - $(stat -c %Y -- "$newest") ))
+
+if [ "$size" -lt "$MIN_BYTES" ]; then
+  echo "FAIL: newest snapshot $newest is only ${size} bytes (min ${MIN_BYTES})." >&2
+  echo "      A 4 KB file here means the empty-SQLite bug has regressed." >&2
+  exit 1
+fi
+
+if [ "$age" -gt "$MAX_AGE" ]; then
+  echo "FAIL: newest snapshot $newest is ${age}s old (max ${MAX_AGE}s)." >&2
+  echo "      k3s has stopped snapshotting even though old files remain." >&2
+  exit 1
+fi
+
+echo "ok: ${count} snapshots on cold, newest $(basename "$newest") (${size} bytes, ${age}s old), ${copied} new this run"
+
+# ---- Retention on the cold copy -------------------------------------------
+# Scoped to the etcd-snapshot-* glob inside $DEST only. Deliberately does NOT
+# match the legacy state-*.db files — those are removed by hand, once, after
+# review. Nothing here touches the restic repos; per CLAUDE.md, restic retention
+# lives exclusively in backup-nas.service.
+if [ "$count" -gt "$KEEP" ]; then
+  tail -n +$((KEEP + 1)) "$listing" | while IFS= read -r old; do
+    [ -n "$old" ] || continue
+    rm -f -- "$old"
+    echo "pruned $(basename "$old")"
+  done
+fi

ansible/templates/backup-etcd.timer.j2

@@ -1,8 +1,15 @@
 [Unit]
-Description=Weekly k3s etcd state backup
+Description=Daily copy of k3s etcd snapshots to cold tier
 
 [Timer]
-OnCalendar=Sun *-*-* 03:00:00
+# Daily, not weekly. k3s retains only ~5 snapshots (2.5 days at its 12h default),
+# so a weekly copy would sample one arbitrary point per week and let the rest age
+# off the NVMe uncopied.
+#
+# 00:30 sits 30 min after k3s' 00:00 snapshot and well before backup-cloud (03:00),
+# which uploads this directory to R2 — so the offsite copy always sees fresh data.
+# The old Sun 03:00 slot collided with backup-cloud.
+OnCalendar=*-*-* 00:30:00
 Persistent=true
 
 [Install]