diff --git a/README.md b/README.md index 9b97db2..e49e051 100644 --- a/README.md +++ b/README.md @@ -258,12 +258,17 @@ alert "fleet-db-backup nightly failed" alert --severity critical "coinpay webhooks are down" # always rings alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring" alert --clear qc-sweep # recovered +alert --check # is the key accepted? ``` smshub sends the same text once per 10 minutes; `--key`/`--every` (default: the message, 6h) keeps a job that fails every minute to one alert per window. If smshub cannot be reached it emails `ALERT_EMAIL` (set in `~/.config/alert/env`) instead. Log: `~/.local/state/alert/alert.log`. +If smshub rejects the key (401/403) the email subject says "smshub alert key +REJECTED", names where the key came from and how to replace it; every real alert +still gets its own email. `alert --check` (daily cron) asks smshub about the key +without alerting anyone and sends that email once per 6h while it is rejected. ### `anthropic-credit-check` diff --git a/bin/alert b/bin/alert index 8219138..86eb9a5 100755 --- a/bin/alert +++ b/bin/alert @@ -5,6 +5,7 @@ # alert --severity critical "coinpay webhooks are down" always rings the phone # alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring" # alert --clear qc-sweep it recovered: the next failure alerts at once +# alert --check is the smshub key accepted? (daily cron) # # Posts to https://smshub.dev/api/v1/alerts with the API key in vault # smshub--prod (SMSHUB_ALERT_API_KEY, Anthony's account), which texts his @@ -13,13 +14,17 @@ # --key/--every (default: the message itself, 6h) keeps a cron job that fails # every minute down to one alert per window. If smshub cannot be reached the # alert is emailed instead (ALERT_EMAIL, default anthony@profullstack.com; set -# it in ~/.config/alert/env), so a failure is never silent. +# it in ~/.config/alert/env), so a failure is never silent. If smshub REJECTS +# the key (401/403) that email says so in its subject: the phone did not ring +# and will not for any alert until the key is replaced. `--check` asks smshub +# about the key without alerting anyone and sends that same email when it is +# rejected, so a revoked key is found by cron, not by the next real outage. # # Exit 0 when sent or deliberately skipped, 1 when nothing went out. # Log: ~/.local/state/alert/alert.log set -uo pipefail -severity=warning key= every=6h to= dry=0 +severity=warning key= every=6h to= dry=0 check=0 while [ $# -gt 0 ]; do case "$1" in --severity) severity=$2; shift 2 ;; @@ -31,14 +36,16 @@ while [ $# -gt 0 ]; do --to) to=$2; shift 2 ;; --to=*) to=${1#*=}; shift ;; --dry-run) dry=1; shift ;; + --check) check=1; shift ;; --clear) rm -f "$HOME/.local/state/alert/$(printf '%s' "$2" | tr -c 'A-Za-z0-9._-' '_').last"; exit 0 ;; - -h|--help) sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + -h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; --) shift; break ;; -*) echo "alert: unknown option $1" >&2; exit 64 ;; *) break ;; esac done msg=$(printf '%s ' "$@" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') +[ "$check" = 1 ] && msg="smshub alert key check" [ -n "$msg" ] || { echo 'usage: alert [--severity info|warning|critical] [--key k --every 6h] "message"' >&2; exit 64; } host=$(hostname -s 2>/dev/null || hostname) case "$msg" in "$host:"*) ;; *) msg="$host: $msg" ;; esac @@ -50,6 +57,7 @@ export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/ STATE="$HOME/.local/state/alert" mkdir -p "$STATE" log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$STATE/alert.log"; } +SMSHUB_URL=${SMSHUB_URL:-https://smshub.dev} seconds() { case "$1" in @@ -57,6 +65,52 @@ seconds() { *m) echo $(( ${1%m} * 60 )) ;; *s) echo "${1%s}" ;; *) echo "$1" ;; esac } +apikey=${SMSHUB_ALERT_API_KEY:-} keysrc='the SMSHUB_ALERT_API_KEY environment variable (not the vault)' +if [ -z "$apikey" ]; then + keysrc='vault smshub--prod SMSHUB_ALERT_API_KEY' + tmp=$(mktemp -d "$STATE/v.XXXXXX") + if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then + apikey=$(python3 -I -c 'import csv,sys +for r in csv.DictReader(open(sys.argv[1])): + if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv") + fi + rm -rf "$tmp" +fi + +email() { # subject body + mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "$1" --via resend --body "$2" >/dev/null 2>&1 +} + +# The key is broken, not smshub: say that loudly, once per 6h however many +# alerts hit it, and name where the key came from and how to replace it. +key_broken() { # http-code response [the alert that did not go out] + log "KEY REJECTED by smshub HTTP $1 (key from $keysrc): ${2:0:200}" + local ks="$STATE/smshub-key-broken.last" + if [ -f "$ks" ] && [ $(( $(date +%s) - $(stat -c %Y "$ks") )) -lt 21600 ] && [ -z "${3:-}" ]; then return 0; fi + email "[alert] $host: smshub alert key REJECTED (HTTP $1): alerts cannot call or text you${3:+ | $3}" "${3:+$3 + +}SMSHUB REJECTED THE ALERT API KEY (HTTP $1: ${2:0:200}). +Until it is replaced no alert from $host can call or text you; each one +falls back to an email like this. + +The key came from $keysrc. +Fix: create a key at $SMSHUB_URL (Settings > API keys, Anthony's account), +then put it in the vault as SMSHUB_ALERT_API_KEY: logicsrc teams pull profullstack +smshub prod --env f.env, edit f.env, logicsrc teams push ... --env f.env, shred f.env. +Check: alert --check" && touch "$ks" +} + +if [ "$check" = 1 ]; then + [ -n "$apikey" ] || { log "check: no key ($keysrc)"; email "[alert] $host: no smshub alert key: alerts cannot call or text you" "alert --check on $host found no key in $keysrc (vault unreachable or key missing)."; exit 1; } + out=$(curl -sS -m 30 -w '\n%{http_code}' "$SMSHUB_URL/api/v1/alerts?limit=1" -H "X-API-Key: $apikey" 2>&1) + code=${out##*$'\n'}; out=${out%$'\n'*} + case "$code" in + 200) rm -f "$STATE/smshub-key-broken.last"; log "check: key ok"; echo "smshub alert key ok"; exit 0 ;; + 401|403) key_broken "$code" "$out"; echo "smshub REJECTED the alert key (HTTP $code)" >&2; exit 1 ;; + *) log "check: smshub HTTP $code (not a key answer): ${out:0:200}"; echo "smshub unreachable or erroring (HTTP $code); key not judged" >&2; exit 1 ;; + esac +fi + [ -n "$key" ] || key=$(printf '%s' "$msg" | sha256sum | cut -c1-16) key=$(printf '%s' "$key" | tr -c 'A-Za-z0-9._-' '_') stamp="$STATE/$key.last" @@ -68,21 +122,10 @@ fi if [ "$dry" = 1 ]; then echo "would alert ($severity): $msg"; exit 0; fi -apikey=${SMSHUB_ALERT_API_KEY:-} -if [ -z "$apikey" ]; then - tmp=$(mktemp -d "$STATE/v.XXXXXX") - if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then - apikey=$(python3 -I -c 'import csv,sys -for r in csv.DictReader(open(sys.argv[1])): - if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv") - fi - rm -rf "$tmp" -fi - body=$(jq -nc --arg m "$msg" --arg s "$severity" --arg t "$to" '{message:$m, severity:$s} + (if $t=="" then {} else {to:$t} end)') out= code=000 if [ -n "$apikey" ]; then - out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST https://smshub.dev/api/v1/alerts \ + out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST "$SMSHUB_URL/api/v1/alerts" \ -H "X-API-Key: $apikey" -H 'Content-Type: application/json' --data "$body" 2>&1) code=${out##*$'\n'} out=${out%$'\n'*} @@ -95,12 +138,17 @@ if [ "$code" = 200 ] || [ "$code" = 201 ]; then fi # smshub said no (or is down): email, so the failure still reaches a person. -reason=${out:-no API key (vault smshub--prod SMSHUB_ALERT_API_KEY)} +if [ "$code" = 401 ] || [ "$code" = 403 ]; then + if key_broken "$code" "$out" "$msg"; then + touch "$stamp"; log "emailed [$key] instead (key rejected): $msg"; exit 0 + fi + log "FAILED [$key] nothing went out: $msg"; exit 1 +fi +reason=${out:-no API key ($keysrc)} log "smshub failed [$key] HTTP $code: $reason" -if mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "[alert] $msg" --via resend \ - --body "$msg +if email "[alert] $msg" "$msg -(smshub alert failed, HTTP $code: ${reason:0:300})" >/dev/null 2>&1; then +(smshub alert failed, HTTP $code: ${reason:0:300})"; then touch "$stamp" log "emailed [$key] instead: $msg" exit 0 diff --git a/bin/scripts-check b/bin/scripts-check index 1b6defb..5c12fdf 100755 --- a/bin/scripts-check +++ b/bin/scripts-check @@ -96,6 +96,16 @@ else note "shellcheck not installed, skipped (apt-get install shellcheck)" fi +# ---------------------------------------------------------------- tests + +# test/*.sh drive a script against fakes (no network, no real email). Each +# exits non-zero on a failed case and skips itself when a tool is missing. +for t in test/*.sh; do + [ -f "$t" ] || continue + note "test $t" + bash "$t" || fail "$t" +done + # ---------------------------------------------------------------- verdict if [ "$rc" = 0 ]; then diff --git a/test/alert.sh b/test/alert.sh new file mode 100755 index 0000000..e162fac --- /dev/null +++ b/test/alert.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# test/alert.sh: bin/alert against a fake smshub. No network, no real email: +# HOME is a temp dir whose ~/.local/bin/mail records what it was asked to send. +# Run by bin/scripts-check. +set -uo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +for t in python3 jq curl; do command -v "$t" >/dev/null || { echo " alert test skipped: no $t"; exit 0; }; done + +tmp=$(mktemp -d); trap 'kill "$srv" 2>/dev/null; rm -rf "$tmp"' EXIT +mkdir -p "$tmp/home/.local/bin" +cat >"$tmp/home/.local/bin/mail" <<'M' +#!/usr/bin/env bash +printf '%s\n' "$@" >>"$HOME/mail.out" +M +chmod +x "$tmp/home/.local/bin/mail" + +# Fake smshub: key "good" is accepted, anything else gets smshub's real 401 body. +cat >"$tmp/srv.py" <<'P' +import http.server, json, sys +class H(http.server.BaseHTTPRequestHandler): + def reply(self): + ok = self.headers.get("X-API-Key") == "good" + n = int(self.headers.get("Content-Length") or 0) + if n: self.rfile.read(n) + body = {"alert": {"channels": ["call"]}, "note": "calling", "alerts": []} if ok else {"error": "Invalid API key"} + self.send_response(200 if ok else 401); self.end_headers(); self.wfile.write(json.dumps(body).encode()) + do_GET = do_POST = reply + def log_message(self, *a): pass +s = http.server.HTTPServer(("127.0.0.1", 0), H); print(s.server_port, flush=True); s.serve_forever() +P +python3 -I "$tmp/srv.py" >"$tmp/port" & srv=$! +for _ in $(seq 50); do [ -s "$tmp/port" ] && break; sleep 0.1; done +export SMSHUB_URL="http://127.0.0.1:$(cat "$tmp/port")" + +fails=0 +ok() { echo " ok: $1"; } +bad() { echo " FAIL: $1" >&2; fails=1; } +run() { HOME="$tmp/home" SMSHUB_ALERT_API_KEY=$1 "$root/bin/alert" "${@:2}" >/dev/null 2>&1; } +mailed() { cat "$tmp/home/mail.out" 2>/dev/null; } + +run good --key t1 "disk full" && [ ! -s "$tmp/home/mail.out" ] && ok "accepted key: sent, no email" || bad "accepted key" + +run bad --key t2 "disk full"; rc=$? +[ $rc = 0 ] && mailed | grep -q 'smshub alert key REJECTED (HTTP 401)' && mailed | grep -q 'disk full' \ + && mailed | grep -q 'environment variable (not the vault)' \ + && ok "401: emailed, subject says key REJECTED, keeps the alert, names the key source" || bad "401 email: $(mailed)" +grep -q 'KEY REJECTED' "$tmp/home/.local/state/alert/alert.log" && ok "401: logged KEY REJECTED" || bad "401 log" + +: >"$tmp/home/mail.out" +run bad --key t3 "second outage" && mailed | grep -q 'second outage' && ok "401 again: every real alert still emailed" || bad "401 second alert" + +: >"$tmp/home/mail.out"; rm -f "$tmp/home/.local/state/alert/smshub-key-broken.last" +run bad --check; rc=$? +[ $rc = 1 ] && mailed | grep -q 'key REJECTED' && ok "--check on rejected key: exit 1 + email" || bad "--check rejected (rc $rc)" +: >"$tmp/home/mail.out" +run bad --check; [ ! -s "$tmp/home/mail.out" ] && ok "--check: one key email per 6h" || bad "--check repeated email" +run good --check && [ ! -f "$tmp/home/.local/state/alert/smshub-key-broken.last" ] && ok "--check on good key: exit 0, clears" || bad "--check good" + +exit $fails