alert + anthropic-credit-check: text/call Anthony on real fleet failures

README.md

@@ -246,6 +246,31 @@ command never stops to authenticate. Scoped to `-u ssh` because a full day is
 Counts `Accepted` only. Failed and half-finished attempts are thousands a day on
 a public box and answer a different question.
 
+### `alert`
+
+Tells Anthony something broke: a text, or a phone call that reads it aloud
+while no number can text yet, through smshub.dev alerts
+(`POST /api/v1/alerts`, Anthony's account, key `SMSHUB_ALERT_API_KEY` in vault
+`smshub--prod`). Cron jobs call it on real failures only.
+
+```sh
+alert "fleet-db-backup nightly failed"
+alert --severity critical "coinpay webhooks are down"    # always rings
+alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring"
+alert --clear qc-sweep                                    # recovered
+```
+
+smshub sends the same text once per 10 minutes; `--key`/`--every` (default:
+the message, 6h) keeps a job that fails every minute to one alert per window.
+If smshub cannot be reached it emails `ALERT_EMAIL` (set in
+`~/.config/alert/env`) instead. Log: `~/.local/state/alert/alert.log`.
+
+### `anthropic-credit-check`
+
+Asks the fleet's shared Anthropic key (vault `profullstack-sharable-keys--prod`)
+for one token. Out of credits, over a usage limit or refused: `alert`, at most
+once a day. Working again: clears it. Hourly from cron; `--dry-run` only prints.
+
 ### `scripts-upgrade`
 
 Moves this box to the newest release of this repo. See [Upgrading](#upgrading).

bin/alert

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# alert: tell Anthony something broke, by text or phone call, through smshub.dev.
+#
+#   alert "fleet-db-backup failed on dev2"
+#   alert --severity critical "coinpay webhooks are down"     always rings the phone
+#   alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring"
+#   alert --clear qc-sweep          it recovered: the next failure alerts at once
+#
+# Posts to https://smshub.dev/api/v1/alerts with the API key in vault
+# smshub--prod (SMSHUB_ALERT_API_KEY, Anthony's account), which texts his
+# alert phone when a number can send SMS and otherwise calls it and reads the
+# message. smshub sends the same text once per 10 minutes; on top of that,
+# --key/--every (default: the message itself, 6h) keeps a cron job that fails
+# every minute down to one alert per window. If smshub cannot be reached the
+# alert is emailed instead (ALERT_EMAIL, default anthony@profullstack.com; set
+# it in ~/.config/alert/env), so a failure is never silent.
+#
+# Exit 0 when sent or deliberately skipped, 1 when nothing went out.
+# Log: ~/.local/state/alert/alert.log
+set -uo pipefail
+
+severity=warning key= every=6h to= dry=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --severity) severity=$2; shift 2 ;;
+    --severity=*) severity=${1#*=}; shift ;;
+    --key) key=$2; shift 2 ;;
+    --key=*) key=${1#*=}; shift ;;
+    --every) every=$2; shift 2 ;;
+    --every=*) every=${1#*=}; shift ;;
+    --to) to=$2; shift 2 ;;
+    --to=*) to=${1#*=}; shift ;;
+    --dry-run) dry=1; shift ;;
+    --clear) rm -f "$HOME/.local/state/alert/$(printf '%s' "$2" | tr -c 'A-Za-z0-9._-' '_').last"; exit 0 ;;
+    -h|--help) sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    --) shift; break ;;
+    -*) echo "alert: unknown option $1" >&2; exit 64 ;;
+    *) break ;;
+  esac
+done
+msg=$(printf '%s ' "$@" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//')
+[ -n "$msg" ] || { echo 'usage: alert [--severity info|warning|critical] [--key k --every 6h] "message"' >&2; exit 64; }
+host=$(hostname -s 2>/dev/null || hostname)
+case "$msg" in "$host:"*) ;; *) msg="$host: $msg" ;; esac
+msg=${msg:0:580}
+
+export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/bin:/bin"
+# shellcheck disable=SC1091
+[ -f "$HOME/.config/alert/env" ] && . "$HOME/.config/alert/env"
+STATE="$HOME/.local/state/alert"
+mkdir -p "$STATE"
+log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$STATE/alert.log"; }
+
+seconds() {
+  case "$1" in
+    *d) echo $(( ${1%d} * 86400 )) ;; *h) echo $(( ${1%h} * 3600 )) ;;
+    *m) echo $(( ${1%m} * 60 )) ;; *s) echo "${1%s}" ;; *) echo "$1" ;;
+  esac
+}
+[ -n "$key" ] || key=$(printf '%s' "$msg" | sha256sum | cut -c1-16)
+key=$(printf '%s' "$key" | tr -c 'A-Za-z0-9._-' '_')
+stamp="$STATE/$key.last"
+window=$(seconds "$every")
+if [ -f "$stamp" ] && [ $(( $(date +%s) - $(stat -c %Y "$stamp") )) -lt "$window" ]; then
+  log "skip [$key] (sent within $every): $msg"
+  exit 0
+fi
+
+if [ "$dry" = 1 ]; then echo "would alert ($severity): $msg"; exit 0; fi
+
+apikey=${SMSHUB_ALERT_API_KEY:-}
+if [ -z "$apikey" ]; then
+  tmp=$(mktemp -d "$STATE/v.XXXXXX")
+  if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then
+    apikey=$(python3 -I -c 'import csv,sys
+for r in csv.DictReader(open(sys.argv[1])):
+    if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv")
+  fi
+  rm -rf "$tmp"
+fi
+
+body=$(jq -nc --arg m "$msg" --arg s "$severity" --arg t "$to" '{message:$m, severity:$s} + (if $t=="" then {} else {to:$t} end)')
+out= code=000
+if [ -n "$apikey" ]; then
+  out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST https://smshub.dev/api/v1/alerts \
+    -H "X-API-Key: $apikey" -H 'Content-Type: application/json' --data "$body" 2>&1)
+  code=${out##*$'\n'}
+  out=${out%$'\n'*}
+fi
+
+if [ "$code" = 200 ] || [ "$code" = 201 ]; then
+  touch "$stamp"
+  log "sent [$key] $severity: $msg -> $(jq -r '(.alert.channels|join("+")) + ": " + .note' <<<"$out" 2>/dev/null || echo "$out")"
+  exit 0
+fi
+
+# smshub said no (or is down): email, so the failure still reaches a person.
+reason=${out:-no API key (vault smshub--prod SMSHUB_ALERT_API_KEY)}
+log "smshub failed [$key] HTTP $code: $reason"
+if mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "[alert] $msg" --via resend \
+     --body "$msg
+
+(smshub alert failed, HTTP $code: ${reason:0:300})" >/dev/null 2>&1; then
+  touch "$stamp"
+  log "emailed [$key] instead: $msg"
+  exit 0
+fi
+log "FAILED [$key] nothing went out: $msg"
+exit 1

bin/anthropic-credit-check

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""anthropic-credit-check: is the fleet's shared Anthropic key still able to answer?
+
+One key (vault profullstack-sharable-keys--prod, ANTHROPIC_API_KEY) is shared by
+about 14 production apps, and each of them falls back silently when it stops
+working. This asks it for one token. When the answer is "credit balance is too
+low", a usage limit or an invalid key, it runs `alert` (at most once a day);
+when it works again it clears that, so the next outage alerts at once.
+
+  anthropic-credit-check            check, alert on failure
+  anthropic-credit-check --dry-run  check and print, never alert
+
+Exit 0 when the key works, 1 when it does not, 2 when the check itself could
+not run (vault or network); only a definite answer from Anthropic alerts.
+Cron: hourly. Log: ~/.local/state/alert/anthropic-credit.log
+"""
+import csv, json, os, subprocess, sys, tempfile, urllib.error, urllib.request
+from datetime import datetime, timezone
+
+HOME = os.path.expanduser("~")
+STATE = f"{HOME}/.local/state/alert"
+KEY_NAME = "anthropic-credit"
+MODEL = "claude-haiku-5-5"
+DRY = "--dry-run" in sys.argv
+os.environ["PATH"] = f"{HOME}/.local/bin:{HOME}/.local/share/mise/shims:/usr/local/bin:/usr/bin:/bin"
+os.makedirs(STATE, exist_ok=True)
+
+
+def log(msg):
+    line = f"{datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')} {msg}"
+    with open(f"{STATE}/anthropic-credit.log", "a") as f:
+        f.write(line + "\n")
+    if DRY or sys.stdout.isatty():
+        print(line)
+
+
+def vault_key():
+    with tempfile.TemporaryDirectory(dir=STATE) as tmp:
+        out = f"{tmp}/v.csv"
+        subprocess.run(["logicsrc", "teams", "export", "profullstack", "profullstack-sharable-keys", "prod",
+                        "-o", out, "--yes"], capture_output=True, timeout=90, check=True)
+        with open(out) as f:
+            for row in csv.DictReader(f):
+                if row["key"] == "ANTHROPIC_API_KEY":
+                    return row["value"]
+    return None
+
+
+def ask(key):
+    req = urllib.request.Request(
+        "https://api.anthropic.com/v1/messages", method="POST",
+        data=json.dumps({"model": MODEL, "max_tokens": 1, "messages": [{"role": "user", "content": "ok"}]}).encode(),
+        headers={"x-api-key": key, "anthropic-version": "2023-06-01", "content-type": "application/json"})
+    try:
+        with urllib.request.urlopen(req, timeout=60) as r:
+            return r.status, ""
+    except urllib.error.HTTPError as e:
+        try:
+            msg = json.loads(e.read()).get("error", {}).get("message", "")
+        except Exception:
+            msg = ""
+        return e.code, msg
+
+
+def main():
+    try:
+        key = vault_key()
+    except Exception as e:
+        log(f"could not read the key from the vault: {e}")
+        return 2
+    if not key:
+        log("no ANTHROPIC_API_KEY in vault profullstack-sharable-keys--prod")
+        return 2
+    try:
+        status, msg = ask(key)
+    except Exception as e:
+        log(f"could not reach Anthropic: {e}")
+        return 2
+
+    if status == 200:
+        log("ok")
+        subprocess.run(["alert", "--clear", KEY_NAME])
+        return 0
+    low = msg.lower()
+    broken = status in (401, 403) or "credit balance" in low or "usage limit" in low or "billing" in low
+    if not broken:
+        # 429 / 5xx / a model name problem: not the outage this watches for.
+        log(f"HTTP {status} (not a credit or key problem, no alert): {msg[:200]}")
+        return 2
+    why = "is out of credits" if "credit balance" in low else "hit its usage limit" if "usage limit" in low else f"is refused (HTTP {status})"
+    text = f"The shared Anthropic API key {why}. About 14 apps have no AI until it is fixed."
+    log(f"HTTP {status}: {msg[:200]}")
+    if not DRY:
+        subprocess.run(["alert", "--key", KEY_NAME, "--every", "24h", text])
+    else:
+        print(f"would alert: {text}")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bin/fleet-nightly

@@ -653,6 +653,10 @@ main().catch(async (e) => {
       const key = process.env.RESEND_API_KEY || loadSecrets().RESEND_API_KEY;
       if (key) await resend(key, { from: FROM, to: TO, subject: 'Fleet nightly FAILED', text: `The nightly fleet report for ${DAY} could not be produced.\n\n${msg}\n\nHost: ${hostname()}` });
     } catch (e2) { console.error(`fleet-nightly: could not mail the failure either: ${e2.message}`); }
+    // And a text or call (bin/alert), since a failure email can fail the same way.
+    try {
+      execFileSync('alert', ['--key', 'fleet-nightly', '--every', '12h', `fleet-nightly failed for ${DAY}: ${String(e && e.message || e).slice(0, 200)}`], { stdio: 'ignore', timeout: 120_000 });
+    } catch { console.error('fleet-nightly: alert failed too'); }
   }
   process.exit(1);
 });

bin/scripts-check

@@ -65,6 +65,7 @@ for f in bin/* install.sh; do
   case "$(head -1 "$f")" in
     *bash*) bash -n "$f" 2>&1 | sed "s/^/  /" ; [ "${PIPESTATUS[0]}" = 0 ] || fail "$f does not parse" ;;
     *node*) node --check "$f" >/dev/null 2>&1 || fail "$f does not parse" ;;
+    *python*) python3 -I -c 'import ast,sys; ast.parse(open(sys.argv[1]).read(), sys.argv[1])' "$f" 2>&1 | sed "s/^/  /" ; [ "${PIPESTATUS[0]}" = 0 ] || fail "$f does not parse" ;;
     *sh*)   sh -n "$f"   2>&1 | sed "s/^/  /" ; [ "${PIPESTATUS[0]}" = 0 ] || fail "$f does not parse" ;;
     *)      note "  no shebang, skipped: $f" ;;
   esac