ralyodio · Oct 17, 2026 · Oct 17, 2026
diff --git a/README.md b/README.md
@@ -258,12 +258,17 @@ alert "fleet-db-backup nightly failed"
 alert --severity critical "coinpay webhooks are down"    # always rings
 alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring"
 alert --clear qc-sweep                                    # recovered
+alert --check                                             # is the key accepted?
 ```
 
 smshub sends the same text once per 10 minutes; `--key`/`--every` (default:
 the message, 6h) keeps a job that fails every minute to one alert per window.
 If smshub cannot be reached it emails `ALERT_EMAIL` (set in
 `~/.config/alert/env`) instead. Log: `~/.local/state/alert/alert.log`.
+If smshub rejects the key (401/403) the email subject says "smshub alert key
+REJECTED", names where the key came from and how to replace it; every real alert
+still gets its own email. `alert --check` (daily cron) asks smshub about the key
+without alerting anyone and sends that email once per 6h while it is rejected.
 
 ### `anthropic-credit-check`
 

diff --git a/bin/alert b/bin/alert
@@ -5,6 +5,7 @@
 #   alert --severity critical "coinpay webhooks are down"     always rings the phone
 #   alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring"
 #   alert --clear qc-sweep          it recovered: the next failure alerts at once
+#   alert --check                   is the smshub key accepted? (daily cron)
 #
 # Posts to https://smshub.dev/api/v1/alerts with the API key in vault
 # smshub--prod (SMSHUB_ALERT_API_KEY, Anthony's account), which texts his
@@ -13,13 +14,17 @@
 # --key/--every (default: the message itself, 6h) keeps a cron job that fails
 # every minute down to one alert per window. If smshub cannot be reached the
 # alert is emailed instead (ALERT_EMAIL, default anthony@profullstack.com; set
-# it in ~/.config/alert/env), so a failure is never silent.
+# it in ~/.config/alert/env), so a failure is never silent. If smshub REJECTS
+# the key (401/403) that email says so in its subject: the phone did not ring
+# and will not for any alert until the key is replaced. `--check` asks smshub
+# about the key without alerting anyone and sends that same email when it is
+# rejected, so a revoked key is found by cron, not by the next real outage.
 #
 # Exit 0 when sent or deliberately skipped, 1 when nothing went out.
 # Log: ~/.local/state/alert/alert.log
 set -uo pipefail
 
-severity=warning key= every=6h to= dry=0
+severity=warning key= every=6h to= dry=0 check=0
 while [ $# -gt 0 ]; do
   case "$1" in
     --severity) severity=$2; shift 2 ;;
@@ -31,14 +36,16 @@ while [ $# -gt 0 ]; do
     --to) to=$2; shift 2 ;;
     --to=*) to=${1#*=}; shift ;;
     --dry-run) dry=1; shift ;;
+    --check) check=1; shift ;;
     --clear) rm -f "$HOME/.local/state/alert/$(printf '%s' "$2" | tr -c 'A-Za-z0-9._-' '_').last"; exit 0 ;;
-    -h|--help) sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    -h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
     --) shift; break ;;
     -*) echo "alert: unknown option $1" >&2; exit 64 ;;
     *) break ;;
   esac
 done
 msg=$(printf '%s ' "$@" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//')
+[ "$check" = 1 ] && msg="smshub alert key check"
 [ -n "$msg" ] || { echo 'usage: alert [--severity info|warning|critical] [--key k --every 6h] "message"' >&2; exit 64; }
 host=$(hostname -s 2>/dev/null || hostname)
 case "$msg" in "$host:"*) ;; *) msg="$host: $msg" ;; esac
@@ -50,13 +57,60 @@ export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/
 STATE="$HOME/.local/state/alert"
 mkdir -p "$STATE"
 log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$STATE/alert.log"; }
+SMSHUB_URL=${SMSHUB_URL:-https://smshub.dev}
 
 seconds() {
   case "$1" in
     *d) echo $(( ${1%d} * 86400 )) ;; *h) echo $(( ${1%h} * 3600 )) ;;
     *m) echo $(( ${1%m} * 60 )) ;; *s) echo "${1%s}" ;; *) echo "$1" ;;
   esac
 }
+apikey=${SMSHUB_ALERT_API_KEY:-} keysrc='the SMSHUB_ALERT_API_KEY environment variable (not the vault)'
+if [ -z "$apikey" ]; then
+  keysrc='vault smshub--prod SMSHUB_ALERT_API_KEY'
+  tmp=$(mktemp -d "$STATE/v.XXXXXX")
+  if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then
+    apikey=$(python3 -I -c 'import csv,sys
+for r in csv.DictReader(open(sys.argv[1])):
+    if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv")
+  fi
+  rm -rf "$tmp"
+fi
+
+email() { # subject body
+  mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "$1" --via resend --body "$2" >/dev/null 2>&1
+}
+
+# The key is broken, not smshub: say that loudly, once per 6h however many
+# alerts hit it, and name where the key came from and how to replace it.
+key_broken() { # http-code response [the alert that did not go out]
+  log "KEY REJECTED by smshub HTTP $1 (key from $keysrc): ${2:0:200}"
+  local ks="$STATE/smshub-key-broken.last"
+  if [ -f "$ks" ] && [ $(( $(date +%s) - $(stat -c %Y "$ks") )) -lt 21600 ] && [ -z "${3:-}" ]; then return 0; fi
+  email "[alert] $host: smshub alert key REJECTED (HTTP $1): alerts cannot call or text you${3:+ | $3}" "${3:+$3
+
+}SMSHUB REJECTED THE ALERT API KEY (HTTP $1: ${2:0:200}).
+Until it is replaced no alert from $host can call or text you; each one
+falls back to an email like this.
+
+The key came from $keysrc.
+Fix: create a key at $SMSHUB_URL (Settings > API keys, Anthony's account),
+then put it in the vault as SMSHUB_ALERT_API_KEY: logicsrc teams pull profullstack
+smshub prod --env f.env, edit f.env, logicsrc teams push ... --env f.env, shred f.env.
+Check: alert --check" && touch "$ks"
+}
+
+if [ "$check" = 1 ]; then
+  [ -n "$apikey" ] || { log "check: no key ($keysrc)"; email "[alert] $host: no smshub alert key: alerts cannot call or text you" "alert --check on $host found no key in $keysrc (vault unreachable or key missing)."; exit 1; }
+  out=$(curl -sS -m 30 -w '\n%{http_code}' "$SMSHUB_URL/api/v1/alerts?limit=1" -H "X-API-Key: $apikey" 2>&1)
+  code=${out##*$'\n'}; out=${out%$'\n'*}
+  case "$code" in
+    200) rm -f "$STATE/smshub-key-broken.last"; log "check: key ok"; echo "smshub alert key ok"; exit 0 ;;
+    401|403) key_broken "$code" "$out"; echo "smshub REJECTED the alert key (HTTP $code)" >&2; exit 1 ;;
+    *) log "check: smshub HTTP $code (not a key answer): ${out:0:200}"; echo "smshub unreachable or erroring (HTTP $code); key not judged" >&2; exit 1 ;;
+  esac
+fi
+
 [ -n "$key" ] || key=$(printf '%s' "$msg" | sha256sum | cut -c1-16)
 key=$(printf '%s' "$key" | tr -c 'A-Za-z0-9._-' '_')
 stamp="$STATE/$key.last"
@@ -68,21 +122,10 @@ fi
 
 if [ "$dry" = 1 ]; then echo "would alert ($severity): $msg"; exit 0; fi
 
-apikey=${SMSHUB_ALERT_API_KEY:-}
-if [ -z "$apikey" ]; then
-  tmp=$(mktemp -d "$STATE/v.XXXXXX")
-  if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then
-    apikey=$(python3 -I -c 'import csv,sys
-for r in csv.DictReader(open(sys.argv[1])):
-    if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv")
-  fi
-  rm -rf "$tmp"
-fi
-
 body=$(jq -nc --arg m "$msg" --arg s "$severity" --arg t "$to" '{message:$m, severity:$s} + (if $t=="" then {} else {to:$t} end)')
 out= code=000
 if [ -n "$apikey" ]; then
-  out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST https://smshub.dev/api/v1/alerts \
+  out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST "$SMSHUB_URL/api/v1/alerts" \
     -H "X-API-Key: $apikey" -H 'Content-Type: application/json' --data "$body" 2>&1)
   code=${out##*$'\n'}
   out=${out%$'\n'*}
@@ -95,12 +138,17 @@ if [ "$code" = 200 ] || [ "$code" = 201 ]; then
 fi
 
 # smshub said no (or is down): email, so the failure still reaches a person.
-reason=${out:-no API key (vault smshub--prod SMSHUB_ALERT_API_KEY)}
+if [ "$code" = 401 ] || [ "$code" = 403 ]; then
+  if key_broken "$code" "$out" "$msg"; then
+    touch "$stamp"; log "emailed [$key] instead (key rejected): $msg"; exit 0
+  fi
+  log "FAILED [$key] nothing went out: $msg"; exit 1
+fi
+reason=${out:-no API key ($keysrc)}
 log "smshub failed [$key] HTTP $code: $reason"
-if mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "[alert] $msg" --via resend \
-     --body "$msg
+if email "[alert] $msg" "$msg
 
-(smshub alert failed, HTTP $code: ${reason:0:300})" >/dev/null 2>&1; then
+(smshub alert failed, HTTP $code: ${reason:0:300})"; then
   touch "$stamp"
   log "emailed [$key] instead: $msg"
   exit 0

diff --git a/bin/scripts-check b/bin/scripts-check
@@ -96,6 +96,16 @@ else
   note "shellcheck not installed, skipped (apt-get install shellcheck)"
 fi
 
+# ---------------------------------------------------------------- tests
+
+# test/*.sh drive a script against fakes (no network, no real email). Each
+# exits non-zero on a failed case and skips itself when a tool is missing.
+for t in test/*.sh; do
+  [ -f "$t" ] || continue
+  note "test $t"
+  bash "$t" || fail "$t"
+done
+
 # ---------------------------------------------------------------- verdict
 
 if [ "$rc" = 0 ]; then

diff --git a/test/alert.sh b/test/alert.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# test/alert.sh: bin/alert against a fake smshub. No network, no real email:
+# HOME is a temp dir whose ~/.local/bin/mail records what it was asked to send.
+# Run by bin/scripts-check.
+set -uo pipefail
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+for t in python3 jq curl; do command -v "$t" >/dev/null || { echo "  alert test skipped: no $t"; exit 0; }; done
+
+tmp=$(mktemp -d); trap 'kill "$srv" 2>/dev/null; rm -rf "$tmp"' EXIT
+mkdir -p "$tmp/home/.local/bin"
+cat >"$tmp/home/.local/bin/mail" <<'M'
+#!/usr/bin/env bash
+printf '%s\n' "$@" >>"$HOME/mail.out"
+M
+chmod +x "$tmp/home/.local/bin/mail"
+
+# Fake smshub: key "good" is accepted, anything else gets smshub's real 401 body.
+cat >"$tmp/srv.py" <<'P'
+import http.server, json, sys
+class H(http.server.BaseHTTPRequestHandler):
+    def reply(self):
+        ok = self.headers.get("X-API-Key") == "good"
+        n = int(self.headers.get("Content-Length") or 0)
+        if n: self.rfile.read(n)
+        body = {"alert": {"channels": ["call"]}, "note": "calling", "alerts": []} if ok else {"error": "Invalid API key"}
+        self.send_response(200 if ok else 401); self.end_headers(); self.wfile.write(json.dumps(body).encode())
+    do_GET = do_POST = reply
+    def log_message(self, *a): pass
+s = http.server.HTTPServer(("127.0.0.1", 0), H); print(s.server_port, flush=True); s.serve_forever()
+P
+python3 -I "$tmp/srv.py" >"$tmp/port" & srv=$!
+for _ in $(seq 50); do [ -s "$tmp/port" ] && break; sleep 0.1; done
+export SMSHUB_URL="http://127.0.0.1:$(cat "$tmp/port")"
+
+fails=0
+ok()  { echo "  ok: $1"; }
+bad() { echo "  FAIL: $1" >&2; fails=1; }
+run() { HOME="$tmp/home" SMSHUB_ALERT_API_KEY=$1 "$root/bin/alert" "${@:2}" >/dev/null 2>&1; }
+mailed() { cat "$tmp/home/mail.out" 2>/dev/null; }
+
+run good --key t1 "disk full" && [ ! -s "$tmp/home/mail.out" ] && ok "accepted key: sent, no email" || bad "accepted key"
+
+run bad --key t2 "disk full"; rc=$?
+[ $rc = 0 ] && mailed | grep -q 'smshub alert key REJECTED (HTTP 401)' && mailed | grep -q 'disk full' \
+  && mailed | grep -q 'environment variable (not the vault)' \
+  && ok "401: emailed, subject says key REJECTED, keeps the alert, names the key source" || bad "401 email: $(mailed)"
+grep -q 'KEY REJECTED' "$tmp/home/.local/state/alert/alert.log" && ok "401: logged KEY REJECTED" || bad "401 log"
+
+: >"$tmp/home/mail.out"
+run bad --key t3 "second outage" && mailed | grep -q 'second outage' && ok "401 again: every real alert still emailed" || bad "401 second alert"
+
+: >"$tmp/home/mail.out"; rm -f "$tmp/home/.local/state/alert/smshub-key-broken.last"
+run bad --check; rc=$?
+[ $rc = 1 ] && mailed | grep -q 'key REJECTED' && ok "--check on rejected key: exit 1 + email" || bad "--check rejected (rc $rc)"
+: >"$tmp/home/mail.out"
+run bad --check; [ ! -s "$tmp/home/mail.out" ] && ok "--check: one key email per 6h" || bad "--check repeated email"
+run good --check && [ ! -f "$tmp/home/.local/state/alert/smshub-key-broken.last" ] && ok "--check on good key: exit 0, clears" || bad "--check good"
+
+exit $fails