diff --git a/ansible/playbooks/apport.yml b/ansible/playbooks/apport.yml new file mode 100644 index 0000000..227a089 --- /dev/null +++ b/ansible/playbooks/apport.yml @@ -0,0 +1,103 @@ +--- +# apport.yml — disable Ubuntu's crash reporter on all Linux hosts. +# +# WHY: on 2026-07-25 apport was found pegged at 100% of a CPU core on n150-2, +# having accumulated 3 weeks and 2 days of CPU time against 26 days of uptime — +# so it had been spinning since roughly three days after boot. It wedged systemd +# badly enough that `systemctl` timed out, `reboot` could not reach init +# ("Failed to talk to init daemon"), and journald logged +# "Failed to send WATCHDOG=1 notification: Transport endpoint is not connected" +# every 90 seconds. k3s was down on the node, taking etcd quorum to 2 of 3 and +# stranding lldap (local-path PVC, cannot reschedule) which took SSO with it. +# Recovery required a SysRq forced reboot. /var/crash was empty, so it was stuck +# on a report it had already cleaned up. +# +# apport provides essentially no value on a headless cluster node — its purpose is +# to prompt a desktop user to file a bug — and it has now caused an outage. This is +# a companion to journald.yml, which caps journal growth after a similar +# self-inflicted incident on 2026-07-23. +# +# Run FROM THE REPO ROOT (n150-1/n150-2 are in kvm_hosts, whose group_vars are +# vaulted, so the vault password is required or fact-gathering fails on exactly +# the two hosts this playbook exists for): +# ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/apport.yml \ +# --vault-password-file ansible/.vault_pass --check --diff +# ansible-playbook -i ansible/inventory/hosts.yml ansible/playbooks/apport.yml \ +# --vault-password-file ansible/.vault_pass +# +# From inside ansible/ the path is just .vault_pass, and ansible.cfg supplies it +# automatically — Ansible only auto-loads ansible.cfg from the current directory, +# which is why repo-root invocations need every flag spelled out. +# +# Limit to a single host: +# ansible-playbook ... -l n150-2 + +- name: Disable the apport crash reporter on all Linux hosts + hosts: all:!x86_nodes:!embedded:!standalone_vms + become: true + gather_facts: true + + tasks: + - name: Skip non-systemd hosts + ansible.builtin.meta: end_host + when: ansible_service_mgr != "systemd" + + - name: Skip non-Debian-family hosts + ansible.builtin.meta: end_host + when: ansible_os_family != "Debian" + + - name: Check whether apport is installed + ansible.builtin.stat: + path: /etc/default/apport + register: apport_defaults + + - name: Skip hosts without apport + ansible.builtin.meta: end_host + when: not apport_defaults.stat.exists + + # Report whether a runaway apport is present right now, so a --check run + # doubles as a fleet-wide audit for the n150-2 failure mode. + - name: Look for a running apport process + ansible.builtin.shell: + cmd: | + set -o pipefail + ps -eo pcpu,etimes,comm --sort=-pcpu | awk '$3 == "apport" {print; found=1} END {exit !found}' + executable: /bin/bash + register: apport_running + changed_when: false + failed_when: false + # Read-only, so run it even under --check. Without this the task is skipped + # in check mode, the registered result has no rc/stdout, and the warning + # below fired with an empty process list — a false alarm that defeats the + # point of using --check as a fleet audit. + check_mode: false + + - name: Warn if apport is currently running + ansible.builtin.debug: + msg: >- + apport is RUNNING on {{ inventory_hostname }}: + {{ apport_running.stdout | trim }} + — disabling the service stops it starting again, but an already-wedged + process must be killed by hand (see docs/RUNBOOK.md). + when: + - apport_running.rc is defined + - apport_running.rc == 0 + - (apport_running.stdout | default('') | trim) != '' + + - name: Disable apport in /etc/default/apport + ansible.builtin.lineinfile: + path: /etc/default/apport + regexp: '^enabled=' + line: 'enabled=0' + mode: "0644" + + - name: Stop and disable the apport service + ansible.builtin.systemd: + name: apport.service + enabled: false + state: stopped + failed_when: false # not present on every Ubuntu variant; the file above is authoritative + + - name: Report result + ansible.builtin.debug: + msg: "apport disabled on {{ inventory_hostname }} ({{ ansible_distribution }} {{ ansible_distribution_version }})" diff --git a/ansible/playbooks/backup-cloud.yml b/ansible/playbooks/backup-cloud.yml index 9245861..fc5b2e9 100644 --- a/ansible/playbooks/backup-cloud.yml +++ b/ansible/playbooks/backup-cloud.yml @@ -46,6 +46,72 @@ owner: root group: root + # ---- Preflight ----------------------------------------------------------- + # Added 2026-07-25. The Vault reads below carry `no_log: true`, which hides the + # *error* as well as the secret — so an expired token surfaces only as + # "censored due to no_log", with no indication of the cause. These three checks + # print no secret values and turn that into a one-line diagnosis. + - name: Preflight — VAULT_TOKEN must be set + ansible.builtin.assert: + that: + - lookup('env', 'VAULT_TOKEN') | length > 0 + fail_msg: >- + VAULT_TOKEN is empty. Pass a current token: + VAULT_TOKEN= ansible-playbook -i ansible/inventory/hosts.yml + ansible/playbooks/backup-cloud.yml + — if running via Semaphore, check the environment on the task template; + tokens expire and the stored one may be stale. + quiet: true + + - name: Preflight — probe Vault reachability and seal status + ansible.builtin.command: + cmd: vault status + environment: + VAULT_ADDR: "{{ vault_addr }}" + register: vault_status_probe + changed_when: false + failed_when: false + check_mode: false # read-only; must run under --check or the assert below is vacuous + # Deliberately NOT no_log — `vault status` prints no secrets, and censoring + # this is precisely what made the expired-token failure undiagnosable. + + - name: Preflight — Vault must be reachable and unsealed + ansible.builtin.assert: + that: + - vault_status_probe.rc is defined + - vault_status_probe.rc == 0 + fail_msg: >- + Vault at {{ vault_addr }} is not usable — `vault status` returned + rc={{ vault_status_probe.rc }} (1 = unreachable or CLI error, 2 = sealed). + If sealed, check vault-unseal.service on rpi5. + {{ vault_status_probe.stderr | default('') }} + quiet: true + + - name: Preflight — token must be able to read the R2 secret + ansible.builtin.shell: + cmd: vault kv get -field=account-id secret/lab/cloudflare-r2 >/dev/null + environment: + VAULT_ADDR: "{{ vault_addr }}" + VAULT_TOKEN: "{{ lookup('env', 'VAULT_TOKEN') }}" + register: vault_read_probe + changed_when: false + failed_when: false + check_mode: false # read-only; must run under --check + # stdout is redirected to /dev/null, so only error text can be captured here. + + - name: Preflight — report token or permission problems clearly + ansible.builtin.assert: + that: + - vault_read_probe.rc is defined + - vault_read_probe.rc == 0 + fail_msg: >- + Could not read secret/lab/cloudflare-r2 (rc={{ vault_read_probe.rc }}). + Likely an expired/revoked VAULT_TOKEN, a policy without read on that path, + or the secret not existing. Check with: + vault token lookup && vault kv list secret/lab + {{ vault_read_probe.stderr | default('') }} + quiet: true + - name: Read R2 credentials from Vault ansible.builtin.shell: cmd: vault kv get -format=json secret/lab/cloudflare-r2 @@ -55,6 +121,7 @@ register: vault_r2_raw no_log: true changed_when: false + check_mode: false - name: Parse R2 credentials ansible.builtin.set_fact: @@ -70,6 +137,7 @@ register: vault_minio_raw no_log: true changed_when: false + check_mode: false - name: Parse MinIO credentials ansible.builtin.set_fact: @@ -108,93 +176,141 @@ group: root content: | #!/bin/bash - set -euo pipefail + # Daily cloud backup of critical cluster state to Cloudflare R2. + # + # ORDERING IS DELIBERATE — read before rearranging. + # + # On 2026-07-25 a wedged crash reporter on n150-2 made one pod + # unreachable. Because this script ran `set -e` with the kubectl-dependent + # dumps FIRST, a single `kubectl cp` failure aborted the entire run — so + # the etcd snapshots and the k3s server token, which are plain file copies + # with no cluster dependency at all and are the most critical items here, + # never made it offsite. One flaky node cost a full day of cluster-state + # backups for a reason unrelated to cluster state. + # + # Now: dependency-free items are staged FIRST, every cluster-dependent dump + # is individually non-fatal, the restic backup ALWAYS runs, and the script + # exits non-zero at the end if anything was missed. A bad node costs you one + # component and an alert instead of everything. + # + # Note the absence of `set -e` — failures are accumulated, not fatal. + set -uo pipefail export KUBECONFIG=/etc/rancher/k3s/k3s.yaml - # STABLE staging path — deliberately NOT mktemp. - # - # This directory is passed to `restic backup` as a source, so its name - # becomes part of the snapshot's path list. With mktemp the name was - # different every run (/tmp/backup-cloud-PX4ZQS, -pOvw3A, ...), and - # since `restic forget` groups by host+paths by default, every snapshot - # formed its own retention group. --keep-daily 7 --keep-weekly 4 - # --keep-monthly 3 then dutifully kept the single member of each group, - # so R2 retention deleted nothing, ever, on a 10 GB free tier. - # A fixed path also means every snapshot has the same tree layout, so - # restore procedures can hardcode paths. See docs/BACKUP-RESTORE.md. - TMPDIR=/var/tmp/backup-cloud-stage - rm -rf "$TMPDIR" - mkdir -p "$TMPDIR" - chmod 700 "$TMPDIR" - trap "rm -rf $TMPDIR" EXIT - - log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; } - - log "Dumping Authelia postgres..." - kubectl exec -n authelia deploy/authelia-postgres -- \ - sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER $POSTGRES_DB' \ - | gzip > "$TMPDIR/authelia.sql.gz" - - log "Dumping Semaphore postgres..." - kubectl exec -n semaphore deploy/semaphore-postgres -- \ - sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER $POSTGRES_DB' \ - | gzip > "$TMPDIR/semaphore.sql.gz" - - log "Dumping Immich postgres..." - kubectl exec -n immich deploy/immich-postgres -- \ - sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER $POSTGRES_DB' \ - | gzip > "$TMPDIR/immich.sql.gz" + # Stable staging path (NOT mktemp, and NOT named TMPDIR — it is passed to + # restic as a source, so its name lands in the snapshot's path list; a + # varying name broke retention grouping, and shadowing TMPDIR would point + # the conventional temp variable at a directory this script rm -rf's). + STAGE=/var/tmp/backup-cloud-stage + rm -rf "$STAGE" + mkdir -p "$STAGE" + chmod 700 "$STAGE" + trap 'rm -rf "$STAGE"' EXIT + + failures=0 + log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; } + warn() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] WARN: $*" >&2; failures=$((failures + 1)); } + + # ---- Tier 1: no external dependencies. Never blocked by a sick node. ---- + + log "Staging k3s server token..." + if install -m 0600 /var/lib/rancher/k3s/server/token "$STAGE/k3s-server-token"; then + log " token staged" + else + warn "could not stage /var/lib/rancher/k3s/server/token — etcd snapshots in this backup will be UNRESTORABLE on new hardware" + fi + + log "Checking etcd snapshots on the cold tier..." + etcd_count=$(find /mnt/cold-8t/k3s-etcd-snapshots -maxdepth 1 -name 'etcd-snapshot-*' -size +1M 2>/dev/null | wc -l) + if [ "$etcd_count" -gt 0 ]; then + log " $etcd_count snapshot(s) >1M present" + else + warn "no etcd snapshots >1M in /mnt/cold-8t/k3s-etcd-snapshots — check backup-etcd.service" + fi + + # ---- Tier 2: needs the cluster. Each failure is isolated. ---- + + dump_pg() { # namespace deployment output-file + local ns="$1" deploy="$2" out="$3" + log "Dumping $ns postgres..." + if kubectl exec -n "$ns" "deploy/$deploy" -- \ + sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER $POSTGRES_DB' \ + | gzip > "$STAGE/$out"; then + # A dump that is well-formed but EMPTY is more dangerous than a + # missing one, because it looks fine: an empty pg_dump still gzips to + # a ~20-byte valid archive that passes both a size test and `gzip -t`. + # Same failure shape as the 4 KB etcd "snapshots" of July 2026, so + # check the DECOMPRESSED size, not the file size. + decompressed=$(gzip -dc "$STAGE/$out" 2>/dev/null | wc -c) + if gzip -t "$STAGE/$out" 2>/dev/null && [ "$decompressed" -ge 100 ]; then + log " $out ok ($(stat -c %s "$STAGE/$out") bytes gz, ${decompressed} raw)" + else + rm -f "$STAGE/$out" + warn "$ns dump was empty or corrupt (${decompressed} bytes decompressed) — discarded" + fi + else + rm -f "$STAGE/$out" + warn "$ns postgres dump failed" + fi + } + + dump_pg authelia authelia-postgres authelia.sql.gz + dump_pg semaphore semaphore-postgres semaphore.sql.gz + dump_pg immich immich-postgres immich.sql.gz log "Copying lldap SQLite database..." LLDAP_POD=$(kubectl get pod -n lldap -l app=lldap \ - -o jsonpath='{.items[0].metadata.name}') - kubectl cp "lldap/${LLDAP_POD}:/data/users.db" "$TMPDIR/lldap.db" + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + if [ -z "$LLDAP_POD" ]; then + warn "no lldap pod found — skipping" + elif kubectl cp "lldap/${LLDAP_POD}:/data/users.db" "$STAGE/lldap.db"; then + log " lldap.db ok ($(stat -c %s "$STAGE/lldap.db" 2>/dev/null || echo 0) bytes)" + else + rm -f "$STAGE/lldap.db" + warn "lldap copy failed — is its node reachable? (kubectl cp needs apiserver->kubelet on :10250)" + fi log "Copying OpenTofu state from MinIO..." - mkdir -p "$TMPDIR/tofu-state" - AWS_ACCESS_KEY_ID="$MINIO_ROOT_USER" \ - AWS_SECRET_ACCESS_KEY="$MINIO_ROOT_PASSWORD" \ - aws s3 cp s3://tofu-state/ "$TMPDIR/tofu-state/" \ - --endpoint-url https://minio.apps.lab.home.arpa \ - --recursive \ - --no-verify-ssl \ - --quiet - - # The k3s server token is REQUIRED to restore any etcd snapshot onto new - # hardware: k3s derives an AES-256 key from it (PBKDF2) to encrypt - # confidential data — CA private keys, bootstrap data — inside the - # datastore itself. Without it the snapshots in this very backup are - # unusable. It was previously backed up nowhere at all; it existed only on - # the three server nodes' local disks, so a total-site loss meant every - # etcd snapshot was decorative. - # - # SECURITY TRADEOFF, read before changing: snapshot + token together is - # full cluster compromise (see docs.k3s.io/cli/etcd-snapshot#security). - # They are co-located here only because this restic repo is encrypted as a - # whole, and its password lives in Vault and on the H4 — not in R2. Anyone - # able to decrypt this repo already holds the keys to everything in it. - # The offline break-glass envelope described in docs/BACKUP-RESTORE.md is - # still the primary recovery path; this is the convenience copy. - log "Copying k3s server token (required to restore etcd snapshots)..." - install -m 0600 /var/lib/rancher/k3s/server/token "$TMPDIR/k3s-server-token" - - # Build source list — include vault-snapshots only if the directory exists + mkdir -p "$STAGE/tofu-state" + if AWS_ACCESS_KEY_ID="$MINIO_ROOT_USER" \ + AWS_SECRET_ACCESS_KEY="$MINIO_ROOT_PASSWORD" \ + aws s3 cp s3://tofu-state/ "$STAGE/tofu-state/" \ + --endpoint-url https://minio.apps.lab.home.arpa \ + --recursive --no-verify-ssl --quiet; then + log " tofu-state ok" + else + warn "OpenTofu state copy from MinIO failed" + fi + + # ---- Always back up whatever we managed to gather ---- + SOURCES=( /mnt/cold-8t/k3s-etcd-snapshots - "$TMPDIR" + "$STAGE" ) [[ -d /mnt/cold-8t/vault-snapshots ]] && SOURCES+=(/mnt/cold-8t/vault-snapshots) log "Running restic backup to R2..." - restic backup "${SOURCES[@]}" --tag cloud --host h4-core + if ! restic backup "${SOURCES[@]}" --tag cloud --host h4-core; then + log "FATAL: restic backup to R2 failed" + exit 1 + fi log "Pruning old cloud snapshots..." # --group-by host,tags (not the default host,paths): all cloud snapshots # share host+tag, so they form ONE retention series. Belt-and-braces - # alongside the stable staging path above — if a future source path ever - # varies again, retention still works. - restic forget --prune --group-by host,tags \ - --keep-daily 7 --keep-weekly 4 --keep-monthly 3 + # alongside the stable staging path above — if a source path ever varies + # again, retention still works. + if ! restic forget --prune --group-by host,tags \ + --keep-daily 7 --keep-weekly 4 --keep-monthly 3; then + warn "retention pass failed — snapshots will accumulate" + fi + + if [ "$failures" -gt 0 ]; then + log "Cloud backup completed with $failures component failure(s) — the backup ran but is INCOMPLETE." + log "hc-ping will be skipped, so healthchecks.io will alert. See warnings above." + exit 1 + fi log "Cloud backup complete." diff --git a/ansible/templates/backup-verify.sh.j2 b/ansible/templates/backup-verify.sh.j2 index a012b4e..2597171 100644 --- a/ansible/templates/backup-verify.sh.j2 +++ b/ansible/templates/backup-verify.sh.j2 @@ -92,7 +92,19 @@ else fi rm -f /tmp/verify-cloud.$$ - etcd_in_cloud=$(restic ls latest "$ETCD_DIR" 2>/dev/null | grep -c 'etcd-snapshot-') + # List the whole snapshot ONCE, then match with path-anchored patterns. + # + # Deliberately NOT using restic's directory-filter form (`restic ls latest `). + # Current restic documents it, but 0.12.1 — what the H4 runs — does not behave the + # same way: on 2026-07-25 the filtered form reported the k3s server token missing + # from a snapshot that demonstrably contained it, while the unfiltered listing + # found it immediately. Anchoring the grep to the full path gives the same + # precision without depending on version-specific filter semantics. + # These cloud snapshots are small (hundreds of entries), so listing all is cheap. + cloud_listing=$(mktemp) + restic ls latest > "$cloud_listing" 2>/dev/null || true + + etcd_in_cloud=$(grep -c "^${ETCD_DIR}/etcd-snapshot-" "$cloud_listing" || true) if [ "${etcd_in_cloud:-0}" -lt 1 ]; then fail "newest R2 snapshot contains NO etcd-snapshot-* files under $ETCD_DIR" else @@ -102,12 +114,14 @@ else # The k3s server token must be in the backup or the etcd snapshots above cannot # be restored onto new hardware — k3s derives the datastore's encryption key # from it. Until 2026-07-25 it was backed up nowhere. - if restic ls latest {{ cloud_stage_dir | quote }} 2>/dev/null | grep -q 'k3s-server-token'; then + if grep -q "^{{ cloud_stage_dir }}/k3s-server-token$" "$cloud_listing"; then pass "newest R2 snapshot contains the k3s server token" else fail "newest R2 snapshot has NO k3s server token — etcd snapshots are unrestorable on new hardware" fi + rm -f "$cloud_listing" + cloud_count=$(restic snapshots --tag cloud --json 2>/dev/null | grep -o '"time":' | wc -l) echo "info: R2 holds ${cloud_count} cloud snapshots" fi diff --git a/docs/BACKUP-RESTORE.md b/docs/BACKUP-RESTORE.md index 13422f8..ead791a 100644 --- a/docs/BACKUP-RESTORE.md +++ b/docs/BACKUP-RESTORE.md @@ -24,7 +24,7 @@ prove any of it works. Supersedes the backup and restore sections of | `backup-vault` | daily 02:30 | Vault raft snapshot (on rpi5) | `/mnt/cold-8t/vault-snapshots/` | 30 days | | `lldap-backup` (k8s CronJob) | daily 02:30 | lldap `users.db` | restic → `/mnt/cold-8t/restic` via NFS | via `backup-nas` policy | | Immich DB dump (k8s CronJob) | daily 01:30 | `pg_dump` | `/mnt/cold-8t/immich/backups/` | captured by `backup-nas` | -| `backup-cloud` | daily 03:00 | etcd snapshots, Vault snapshots, lldap DB, Postgres dumps (Authelia/Immich/Semaphore), OpenTofu state, **k3s server token** | restic → Cloudflare R2 `homelab-backup` | 7d / 4w / 3m | +| `backup-cloud` | daily 03:00 | **k3s server token** and etcd snapshots first (no cluster dependency), then Vault snapshots, lldap DB, Postgres dumps (Authelia/Immich/Semaphore), OpenTofu state | restic → Cloudflare R2 `homelab-backup` | 7d / 4w / 3m | | `backup-verify` | weekly Sun 04:00 | *verification only, read-only* | — | — | `backup-offsite.timer` also exists but is a **no-op** — `restic_offsite_repo` @@ -68,6 +68,26 @@ Anyone holding both can extract every secret and the cluster CA private keys. Th are co-located in the R2 repo only because that repo is encrypted as a whole and its password is not stored in R2. Treat the R2 restic password as a crown jewel. +### A partial cloud backup is normal, and it tells you so + +`backup-cloud` is deliberately **not** all-or-nothing. Items with no cluster +dependency — the k3s server token and the etcd snapshots — are staged first, and +each `kubectl`-dependent dump is individually non-fatal. The restic backup always +runs; the script exits non-zero at the end if anything was missed, which suppresses +the healthchecks.io ping so you get told. + +So a failure here means *"the backup ran but is incomplete"*, not *"there is no +backup"*. Read the `WARN:` lines to see which component is missing. + +This shape exists because of 2026-07-25: a wedged crash reporter on n150-2 made one +pod unreachable, and because the dumps ran first under `set -e`, a single +`kubectl cp` failure aborted the whole run — costing a day of offsite cluster-state +backups for a reason that had nothing to do with cluster state. + +Postgres dumps are checked by **decompressed** size, not file size. An empty +`pg_dump` still gzips to a ~20-byte archive that passes `gzip -t` — the same +well-formed-but-empty shape as the 4 KB etcd snapshots. + ### The break-glass envelope Recovery from total loss needs credentials that are themselves stored inside the diff --git a/docs/REVIEW-2026-07-24.md b/docs/REVIEW-2026-07-24.md index 9058205..6a4ef16 100644 --- a/docs/REVIEW-2026-07-24.md +++ b/docs/REVIEW-2026-07-24.md @@ -26,8 +26,16 @@ Three themes: ## Critical > **Update 2026-07-25 — verification pass run on the H4.** C1 **confirmed** (worse than -> written). C6 **largely disproved** — the cold-sec copy is working. C7 **downgraded** — -> never committed. H20 **confirmed**. See the inline notes on each. +> written) and since **fixed**. C6 **disproved** — the cold-sec copy works. C7 +> **downgraded** — never committed. H20 **confirmed**. H11 **retracted** — the +> PolicyExceptions were never committed either. C8 **added** — the k3s server token was +> backed up nowhere, making every offsite etcd snapshot unrestorable. See inline notes. +> +> **A caveat on this document's reliability.** It was written against a *copy* of the +> repo with no `.git`, so every claim of the form "X is committed" was an inference from +> a file existing on disk. Two of them (C6, H11) turned out to be wrong in exactly that +> way. Findings that depend on tracked-vs-untracked status should be re-checked with +> `git log -- ` on the H4 before being acted on. ### C1. `backup-etcd` snapshots a file that hasn't existed since the HA migration — PATCHED @@ -247,7 +255,7 @@ directory carries the password. `LAB-URLS.md` has the same shape of problem | H8 | Vault is contacted over plaintext HTTP with a static, never-rotated token that isn't in git (so the store is unreproducible) | `gitops/workloads/immich/external-secret.yaml:20` | | H9 | PodSecurityAdmission is documented but no namespace carries `pod-security.kubernetes.io/*` labels — everything is at k3s default (privileged). The only gate is Kyverno, whose `webhookFailurePolicy: Ignore` fails **open** | `gitops/apps/kyverno.yaml:35`, all `namespace.yaml` | | H10 | Stateful workloads with `local-path` PVCs and no `nodeSelector` can land on the arm64 OPi agents (which are labelled but **not tainted**) and get pinned there by PV node affinity | `immich/*.yaml`, `authelia/*.yaml`, `lldap/deployment.yaml`, `home-assistant/deployment.yaml` | -| H11 | Kyverno PolicyExceptions for ollama and whisper are still committed and inside Argo-synced directories with `selfHeal` — deleting them imperatively means Argo recreated them. Both images are now pinned, so they're pure attack surface. `TODO-2026-07-23.md:163` claiming "0 PolicyExceptions" is false | `ai-backends/policy-exceptions.yaml`, `whisper/policy-exceptions.yaml` | +| ~~H11~~ | **WRONG — retracted 2026-07-25.** I reported the ollama/whisper PolicyExceptions as committed and recreated by Argo. `git status` on the H4 shows both files **untracked**: they were never committed, so Argo (which syncs from the remote) never saw them and `TODO-2026-07-23.md:163`'s "0 PolicyExceptions" was correct. Same root cause as the C6 error — the mounted review copy has no `.git`, so "present on disk" was indistinguishable from "tracked". Action is merely to delete the stale local files. | `ai-backends/policy-exceptions.yaml`, `whisper/policy-exceptions.yaml` | ### Hosts / automation @@ -282,6 +290,26 @@ directory carries the password. `LAB-URLS.md` has the same shape of problem ## Medium +- **M-new-5: `opi-zero2w-4` (192.168.1.99) is down and nothing noticed.** *(found 2026-07-25)* + `No route to host` on every Ansible run. That is the **secondary MQTT broker** — the HA + bridge peer for `opi-zero2w-2` described in `README.md:100` and `ARCHITECTURE.md:180-181` + as providing automatic M5Stack failover. The failover target is itself unavailable, so + MQTT is currently a single point of failure while the documentation says otherwise. + Unknown how long; nothing alerts on it. It is also absent from the Prometheus static + scrape list (see H28). Same shape as the apport incident: a real degradation running + invisibly for an unknown period. +- **M-new-6: `xu3-1` runs Ubuntu 16.04.** *(found 2026-07-25)* EOL since April 2021 — no + security updates for five years. Only a build agent, but it holds the Ansible SSH key and + sits on the same flat network as everything else. Either rebuild it on a supported + release or retire it; `docs/HARDWARE.md` should say which. +- **M-new-7: `ansible.cfg` is only auto-loaded from `ansible/`.** *(found 2026-07-25)* + Ansible reads `ansible.cfg` from the current directory, so repo-root invocations silently + ignore `ansible/ansible.cfg` — no `vault_password_file`, no `roles_path`. Every flag then + has to be passed by hand, and forgetting `--vault-password-file` fails fact-gathering on + precisely the vaulted hosts (`kvm_hosts`) you were targeting. Fix with a thin root + `ansible.cfg`, or `ANSIBLE_CONFIG=ansible/ansible.cfg` in the Makefile. Related to the + absolute `roles_path` already noted in the LOW section. + - **M-new-1: nothing ever trims the cold-sec copy repo.** *(found 2026-07-25)* Retention runs only against the primary (`backup-nas.service.j2:15`); the copy accumulates forever — 40 snapshots vs the primary's 19 already. `CLAUDE.md` correctly forbids running `restic forget`