From c9ecd217fb74d77a49163dbdcb136fe0bef8728b Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 17 Oct 2026 22:10:33 +0000 Subject: [PATCH] alert: say loudly when smshub rejects the key, add --check + tests A 401/403 from smshub now emails with the subject "smshub alert key REJECTED", names where the key came from (env override vs vault) and how to replace it, and logs KEY REJECTED. alert --check asks smshub about the key without alerting anyone (daily cron), emailing once per 6h while it is rejected. SMSHUB_URL is overridable so test/alert.sh can drive a fake smshub; scripts-check runs test/*.sh. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +++ bin/alert | 86 ++++++++++++++++++++++++++++++++++++----------- bin/scripts-check | 10 ++++++ test/alert.sh | 59 ++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 19 deletions(-) create mode 100755 test/alert.sh diff --git a/README.md b/README.md index 9b97db2..e49e051 100644 --- a/README.md +++ b/README.md @@ -258,12 +258,17 @@ alert "fleet-db-backup nightly failed" alert --severity critical "coinpay webhooks are down" # always rings alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring" alert --clear qc-sweep # recovered +alert --check # is the key accepted? ``` smshub sends the same text once per 10 minutes; `--key`/`--every` (default: the message, 6h) keeps a job that fails every minute to one alert per window. If smshub cannot be reached it emails `ALERT_EMAIL` (set in `~/.config/alert/env`) instead. Log: `~/.local/state/alert/alert.log`. +If smshub rejects the key (401/403) the email subject says "smshub alert key +REJECTED", names where the key came from and how to replace it; every real alert +still gets its own email. `alert --check` (daily cron) asks smshub about the key +without alerting anyone and sends that email once per 6h while it is rejected. ### `anthropic-credit-check` diff --git a/bin/alert b/bin/alert index 8219138..86eb9a5 100755 --- a/bin/alert +++ b/bin/alert @@ -5,6 +5,7 @@ # alert --severity critical "coinpay webhooks are down" always rings the phone # alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring" # alert --clear qc-sweep it recovered: the next failure alerts at once +# alert --check is the smshub key accepted? (daily cron) # # Posts to https://smshub.dev/api/v1/alerts with the API key in vault # smshub--prod (SMSHUB_ALERT_API_KEY, Anthony's account), which texts his @@ -13,13 +14,17 @@ # --key/--every (default: the message itself, 6h) keeps a cron job that fails # every minute down to one alert per window. If smshub cannot be reached the # alert is emailed instead (ALERT_EMAIL, default anthony@profullstack.com; set -# it in ~/.config/alert/env), so a failure is never silent. +# it in ~/.config/alert/env), so a failure is never silent. If smshub REJECTS +# the key (401/403) that email says so in its subject: the phone did not ring +# and will not for any alert until the key is replaced. `--check` asks smshub +# about the key without alerting anyone and sends that same email when it is +# rejected, so a revoked key is found by cron, not by the next real outage. # # Exit 0 when sent or deliberately skipped, 1 when nothing went out. # Log: ~/.local/state/alert/alert.log set -uo pipefail -severity=warning key= every=6h to= dry=0 +severity=warning key= every=6h to= dry=0 check=0 while [ $# -gt 0 ]; do case "$1" in --severity) severity=$2; shift 2 ;; @@ -31,14 +36,16 @@ while [ $# -gt 0 ]; do --to) to=$2; shift 2 ;; --to=*) to=${1#*=}; shift ;; --dry-run) dry=1; shift ;; + --check) check=1; shift ;; --clear) rm -f "$HOME/.local/state/alert/$(printf '%s' "$2" | tr -c 'A-Za-z0-9._-' '_').last"; exit 0 ;; - -h|--help) sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + -h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; --) shift; break ;; -*) echo "alert: unknown option $1" >&2; exit 64 ;; *) break ;; esac done msg=$(printf '%s ' "$@" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') +[ "$check" = 1 ] && msg="smshub alert key check" [ -n "$msg" ] || { echo 'usage: alert [--severity info|warning|critical] [--key k --every 6h] "message"' >&2; exit 64; } host=$(hostname -s 2>/dev/null || hostname) case "$msg" in "$host:"*) ;; *) msg="$host: $msg" ;; esac @@ -50,6 +57,7 @@ export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/ STATE="$HOME/.local/state/alert" mkdir -p "$STATE" log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$STATE/alert.log"; } +SMSHUB_URL=${SMSHUB_URL:-https://smshub.dev} seconds() { case "$1" in @@ -57,6 +65,52 @@ seconds() { *m) echo $(( ${1%m} * 60 )) ;; *s) echo "${1%s}" ;; *) echo "$1" ;; esac } +apikey=${SMSHUB_ALERT_API_KEY:-} keysrc='the SMSHUB_ALERT_API_KEY environment variable (not the vault)' +if [ -z "$apikey" ]; then + keysrc='vault smshub--prod SMSHUB_ALERT_API_KEY' + tmp=$(mktemp -d "$STATE/v.XXXXXX") + if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then + apikey=$(python3 -I -c 'import csv,sys +for r in csv.DictReader(open(sys.argv[1])): + if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv") + fi + rm -rf "$tmp" +fi + +email() { # subject body + mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "$1" --via resend --body "$2" >/dev/null 2>&1 +} + +# The key is broken, not smshub: say that loudly, once per 6h however many +# alerts hit it, and name where the key came from and how to replace it. +key_broken() { # http-code response [the alert that did not go out] + log "KEY REJECTED by smshub HTTP $1 (key from $keysrc): ${2:0:200}" + local ks="$STATE/smshub-key-broken.last" + if [ -f "$ks" ] && [ $(( $(date +%s) - $(stat -c %Y "$ks") )) -lt 21600 ] && [ -z "${3:-}" ]; then return 0; fi + email "[alert] $host: smshub alert key REJECTED (HTTP $1): alerts cannot call or text you${3:+ | $3}" "${3:+$3 + +}SMSHUB REJECTED THE ALERT API KEY (HTTP $1: ${2:0:200}). +Until it is replaced no alert from $host can call or text you; each one +falls back to an email like this. + +The key came from $keysrc. +Fix: create a key at $SMSHUB_URL (Settings > API keys, Anthony's account), +then put it in the vault as SMSHUB_ALERT_API_KEY: logicsrc teams pull profullstack +smshub prod --env f.env, edit f.env, logicsrc teams push ... --env f.env, shred f.env. +Check: alert --check" && touch "$ks" +} + +if [ "$check" = 1 ]; then + [ -n "$apikey" ] || { log "check: no key ($keysrc)"; email "[alert] $host: no smshub alert key: alerts cannot call or text you" "alert --check on $host found no key in $keysrc (vault unreachable or key missing)."; exit 1; } + out=$(curl -sS -m 30 -w '\n%{http_code}' "$SMSHUB_URL/api/v1/alerts?limit=1" -H "X-API-Key: $apikey" 2>&1) + code=${out##*$'\n'}; out=${out%$'\n'*} + case "$code" in + 200) rm -f "$STATE/smshub-key-broken.last"; log "check: key ok"; echo "smshub alert key ok"; exit 0 ;; + 401|403) key_broken "$code" "$out"; echo "smshub REJECTED the alert key (HTTP $code)" >&2; exit 1 ;; + *) log "check: smshub HTTP $code (not a key answer): ${out:0:200}"; echo "smshub unreachable or erroring (HTTP $code); key not judged" >&2; exit 1 ;; + esac +fi + [ -n "$key" ] || key=$(printf '%s' "$msg" | sha256sum | cut -c1-16) key=$(printf '%s' "$key" | tr -c 'A-Za-z0-9._-' '_') stamp="$STATE/$key.last" @@ -68,21 +122,10 @@ fi if [ "$dry" = 1 ]; then echo "would alert ($severity): $msg"; exit 0; fi -apikey=${SMSHUB_ALERT_API_KEY:-} -if [ -z "$apikey" ]; then - tmp=$(mktemp -d "$STATE/v.XXXXXX") - if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then - apikey=$(python3 -I -c 'import csv,sys -for r in csv.DictReader(open(sys.argv[1])): - if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv") - fi - rm -rf "$tmp" -fi - body=$(jq -nc --arg m "$msg" --arg s "$severity" --arg t "$to" '{message:$m, severity:$s} + (if $t=="" then {} else {to:$t} end)') out= code=000 if [ -n "$apikey" ]; then - out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST https://smshub.dev/api/v1/alerts \ + out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST "$SMSHUB_URL/api/v1/alerts" \ -H "X-API-Key: $apikey" -H 'Content-Type: application/json' --data "$body" 2>&1) code=${out##*$'\n'} out=${out%$'\n'*} @@ -95,12 +138,17 @@ if [ "$code" = 200 ] || [ "$code" = 201 ]; then fi # smshub said no (or is down): email, so the failure still reaches a person. -reason=${out:-no API key (vault smshub--prod SMSHUB_ALERT_API_KEY)} +if [ "$code" = 401 ] || [ "$code" = 403 ]; then + if key_broken "$code" "$out" "$msg"; then + touch "$stamp"; log "emailed [$key] instead (key rejected): $msg"; exit 0 + fi + log "FAILED [$key] nothing went out: $msg"; exit 1 +fi +reason=${out:-no API key ($keysrc)} log "smshub failed [$key] HTTP $code: $reason" -if mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "[alert] $msg" --via resend \ - --body "$msg +if email "[alert] $msg" "$msg -(smshub alert failed, HTTP $code: ${reason:0:300})" >/dev/null 2>&1; then +(smshub alert failed, HTTP $code: ${reason:0:300})"; then touch "$stamp" log "emailed [$key] instead: $msg" exit 0 diff --git a/bin/scripts-check b/bin/scripts-check index 1b6defb..5c12fdf 100755 --- a/bin/scripts-check +++ b/bin/scripts-check @@ -96,6 +96,16 @@ else note "shellcheck not installed, skipped (apt-get install shellcheck)" fi +# ---------------------------------------------------------------- tests + +# test/*.sh drive a script against fakes (no network, no real email). Each +# exits non-zero on a failed case and skips itself when a tool is missing. +for t in test/*.sh; do + [ -f "$t" ] || continue + note "test $t" + bash "$t" || fail "$t" +done + # ---------------------------------------------------------------- verdict if [ "$rc" = 0 ]; then diff --git a/test/alert.sh b/test/alert.sh new file mode 100755 index 0000000..e162fac --- /dev/null +++ b/test/alert.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# test/alert.sh: bin/alert against a fake smshub. No network, no real email: +# HOME is a temp dir whose ~/.local/bin/mail records what it was asked to send. +# Run by bin/scripts-check. +set -uo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +for t in python3 jq curl; do command -v "$t" >/dev/null || { echo " alert test skipped: no $t"; exit 0; }; done + +tmp=$(mktemp -d); trap 'kill "$srv" 2>/dev/null; rm -rf "$tmp"' EXIT +mkdir -p "$tmp/home/.local/bin" +cat >"$tmp/home/.local/bin/mail" <<'M' +#!/usr/bin/env bash +printf '%s\n' "$@" >>"$HOME/mail.out" +M +chmod +x "$tmp/home/.local/bin/mail" + +# Fake smshub: key "good" is accepted, anything else gets smshub's real 401 body. +cat >"$tmp/srv.py" <<'P' +import http.server, json, sys +class H(http.server.BaseHTTPRequestHandler): + def reply(self): + ok = self.headers.get("X-API-Key") == "good" + n = int(self.headers.get("Content-Length") or 0) + if n: self.rfile.read(n) + body = {"alert": {"channels": ["call"]}, "note": "calling", "alerts": []} if ok else {"error": "Invalid API key"} + self.send_response(200 if ok else 401); self.end_headers(); self.wfile.write(json.dumps(body).encode()) + do_GET = do_POST = reply + def log_message(self, *a): pass +s = http.server.HTTPServer(("127.0.0.1", 0), H); print(s.server_port, flush=True); s.serve_forever() +P +python3 -I "$tmp/srv.py" >"$tmp/port" & srv=$! +for _ in $(seq 50); do [ -s "$tmp/port" ] && break; sleep 0.1; done +export SMSHUB_URL="http://127.0.0.1:$(cat "$tmp/port")" + +fails=0 +ok() { echo " ok: $1"; } +bad() { echo " FAIL: $1" >&2; fails=1; } +run() { HOME="$tmp/home" SMSHUB_ALERT_API_KEY=$1 "$root/bin/alert" "${@:2}" >/dev/null 2>&1; } +mailed() { cat "$tmp/home/mail.out" 2>/dev/null; } + +run good --key t1 "disk full" && [ ! -s "$tmp/home/mail.out" ] && ok "accepted key: sent, no email" || bad "accepted key" + +run bad --key t2 "disk full"; rc=$? +[ $rc = 0 ] && mailed | grep -q 'smshub alert key REJECTED (HTTP 401)' && mailed | grep -q 'disk full' \ + && mailed | grep -q 'environment variable (not the vault)' \ + && ok "401: emailed, subject says key REJECTED, keeps the alert, names the key source" || bad "401 email: $(mailed)" +grep -q 'KEY REJECTED' "$tmp/home/.local/state/alert/alert.log" && ok "401: logged KEY REJECTED" || bad "401 log" + +: >"$tmp/home/mail.out" +run bad --key t3 "second outage" && mailed | grep -q 'second outage' && ok "401 again: every real alert still emailed" || bad "401 second alert" + +: >"$tmp/home/mail.out"; rm -f "$tmp/home/.local/state/alert/smshub-key-broken.last" +run bad --check; rc=$? +[ $rc = 1 ] && mailed | grep -q 'key REJECTED' && ok "--check on rejected key: exit 1 + email" || bad "--check rejected (rc $rc)" +: >"$tmp/home/mail.out" +run bad --check; [ ! -s "$tmp/home/mail.out" ] && ok "--check: one key email per 6h" || bad "--check repeated email" +run good --check && [ ! -f "$tmp/home/.local/state/alert/smshub-key-broken.last" ] && ok "--check on good key: exit 0, clears" || bad "--check good" + +exit $fails