From a8663a5233ee8d24f116ef272ffea0d30d69a194 Mon Sep 17 00:00:00 2001
From: Scott Wares <waresscott@gmail.com>
Date: Sat, 25 Jul 2026 19:16:32 +0000
Subject: [PATCH] fix(monitoring): add the ExternalSecret PR #NNN missed

alertmanagerSpec.secrets referenced alertmanager-endpoints, but the
ExternalSecret that creates it was a new file and never got staged
(git commit -am only stages tracked files). Alertmanager has been stuck
on FailedMount since the merge, so the cluster currently has no
alerting at all.
---
 .../alertmanager-externalsecret.yaml          | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 gitops/workloads/monitoring/alertmanager-externalsecret.yaml

diff --git a/gitops/workloads/monitoring/alertmanager-externalsecret.yaml b/gitops/workloads/monitoring/alertmanager-externalsecret.yaml
new file mode 100644
index 0000000..2b8dd28
--- /dev/null
+++ b/gitops/workloads/monitoring/alertmanager-externalsecret.yaml
@@ -0,0 +1,49 @@
+---
+# Alertmanager notification endpoints, pulled from Vault.
+#
+# WHY THESE ARE SECRETS
+#
+# ntfy has no accounts — the topic name IS the password. Anyone who knows the
+# topic can both read every alert this lab emits and publish fake ones. Same for
+# the healthchecks.io ping URL: its UUID is the only credential. Neither may be
+# committed, so both live in Vault and are mounted into the Alertmanager pod.
+#
+# Storing the FULL URL (not just the topic) also means the query string —
+# including ntfy's message-template parameters, priority and tags — can be tuned
+# without a commit and an Argo sync. Alertmanager reads these files at
+# /etc/alertmanager/secrets/alertmanager-endpoints/ via `url_file`.
+#
+# SEED VAULT BEFORE ARGO SYNCS THIS:
+#
+#   # Pick an unguessable topic — this is the password.
+#   TOPIC="lab-$(openssl rand -hex 16)"
+#
+#   vault kv patch secret/lab/alertmanager \
+#     ntfy-url="https://ntfy.sh/${TOPIC}?priority=5&tags=rotating_light" \
+#     watchdog-url="https://hc-ping.com/<uuid-of-the-alertmanager-watchdog-check>"
+#
+# Then subscribe to ${TOPIC} in the ntfy app, and create a healthchecks.io check
+# named `alertmanager-watchdog` with Period 15m / Grace 10m — see the Watchdog
+# receiver in gitops/apps/monitoring.yaml for why that cadence.
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: alertmanager-endpoints
+  namespace: monitoring
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  target:
+    name: alertmanager-endpoints
+    creationPolicy: Owner
+  data:
+    - secretKey: ntfy-url
+      remoteRef:
+        key: secret/lab/alertmanager
+        property: ntfy-url
+    - secretKey: watchdog-url
+      remoteRef:
+        key: secret/lab/alertmanager
+        property: watchdog-url