From 19a4548a83e94761cb319b5e2f78cb7db0b41083 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 17 Oct 2026 16:04:32 +0000 Subject: [PATCH] alert + anthropic-credit-check: text/call Anthony on real fleet failures bin/alert posts to smshub.dev alerts (text when a number can send SMS, otherwise a call that reads it), key from vault smshub--prod; --key/--every keeps a failing cron to one alert per window, --clear on recovery, and it emails ALERT_EMAIL when smshub is unreachable. bin/anthropic-credit-check asks the shared Anthropic key for one token hourly and alerts (once a day) on out-of-credits / usage limit / refused. fleet-nightly alerts on a failed run, next to the failure email. scripts-check now parses python scripts too. Co-Authored-By: Claude Opus 5.5 --- README.md | 25 +++++++++ bin/alert | 109 +++++++++++++++++++++++++++++++++++++ bin/anthropic-credit-check | 101 ++++++++++++++++++++++++++++++++++ bin/fleet-nightly | 4 ++ bin/scripts-check | 1 + 5 files changed, 240 insertions(+) create mode 100755 bin/alert create mode 100755 bin/anthropic-credit-check diff --git a/README.md b/README.md index 8a6bc96..9b97db2 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,31 @@ command never stops to authenticate. Scoped to `-u ssh` because a full day is Counts `Accepted` only. Failed and half-finished attempts are thousands a day on a public box and answer a different question. +### `alert` + +Tells Anthony something broke: a text, or a phone call that reads it aloud +while no number can text yet, through smshub.dev alerts +(`POST /api/v1/alerts`, Anthony's account, key `SMSHUB_ALERT_API_KEY` in vault +`smshub--prod`). Cron jobs call it on real failures only. + +```sh +alert "fleet-db-backup nightly failed" +alert --severity critical "coinpay webhooks are down" # always rings +alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring" +alert --clear qc-sweep # recovered +``` + +smshub sends the same text once per 10 minutes; `--key`/`--every` (default: +the message, 6h) keeps a job that fails every minute to one alert per window. +If smshub cannot be reached it emails `ALERT_EMAIL` (set in +`~/.config/alert/env`) instead. Log: `~/.local/state/alert/alert.log`. + +### `anthropic-credit-check` + +Asks the fleet's shared Anthropic key (vault `profullstack-sharable-keys--prod`) +for one token. Out of credits, over a usage limit or refused: `alert`, at most +once a day. Working again: clears it. Hourly from cron; `--dry-run` only prints. + ### `scripts-upgrade` Moves this box to the newest release of this repo. See [Upgrading](#upgrading). diff --git a/bin/alert b/bin/alert new file mode 100755 index 0000000..8219138 --- /dev/null +++ b/bin/alert @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# alert: tell Anthony something broke, by text or phone call, through smshub.dev. +# +# alert "fleet-db-backup failed on dev2" +# alert --severity critical "coinpay webhooks are down" always rings the phone +# alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring" +# alert --clear qc-sweep it recovered: the next failure alerts at once +# +# Posts to https://smshub.dev/api/v1/alerts with the API key in vault +# smshub--prod (SMSHUB_ALERT_API_KEY, Anthony's account), which texts his +# alert phone when a number can send SMS and otherwise calls it and reads the +# message. smshub sends the same text once per 10 minutes; on top of that, +# --key/--every (default: the message itself, 6h) keeps a cron job that fails +# every minute down to one alert per window. If smshub cannot be reached the +# alert is emailed instead (ALERT_EMAIL, default anthony@profullstack.com; set +# it in ~/.config/alert/env), so a failure is never silent. +# +# Exit 0 when sent or deliberately skipped, 1 when nothing went out. +# Log: ~/.local/state/alert/alert.log +set -uo pipefail + +severity=warning key= every=6h to= dry=0 +while [ $# -gt 0 ]; do + case "$1" in + --severity) severity=$2; shift 2 ;; + --severity=*) severity=${1#*=}; shift ;; + --key) key=$2; shift 2 ;; + --key=*) key=${1#*=}; shift ;; + --every) every=$2; shift 2 ;; + --every=*) every=${1#*=}; shift ;; + --to) to=$2; shift 2 ;; + --to=*) to=${1#*=}; shift ;; + --dry-run) dry=1; shift ;; + --clear) rm -f "$HOME/.local/state/alert/$(printf '%s' "$2" | tr -c 'A-Za-z0-9._-' '_').last"; exit 0 ;; + -h|--help) sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + --) shift; break ;; + -*) echo "alert: unknown option $1" >&2; exit 64 ;; + *) break ;; + esac +done +msg=$(printf '%s ' "$@" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//') +[ -n "$msg" ] || { echo 'usage: alert [--severity info|warning|critical] [--key k --every 6h] "message"' >&2; exit 64; } +host=$(hostname -s 2>/dev/null || hostname) +case "$msg" in "$host:"*) ;; *) msg="$host: $msg" ;; esac +msg=${msg:0:580} + +export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/bin:/bin" +# shellcheck disable=SC1091 +[ -f "$HOME/.config/alert/env" ] && . "$HOME/.config/alert/env" +STATE="$HOME/.local/state/alert" +mkdir -p "$STATE" +log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$STATE/alert.log"; } + +seconds() { + case "$1" in + *d) echo $(( ${1%d} * 86400 )) ;; *h) echo $(( ${1%h} * 3600 )) ;; + *m) echo $(( ${1%m} * 60 )) ;; *s) echo "${1%s}" ;; *) echo "$1" ;; + esac +} +[ -n "$key" ] || key=$(printf '%s' "$msg" | sha256sum | cut -c1-16) +key=$(printf '%s' "$key" | tr -c 'A-Za-z0-9._-' '_') +stamp="$STATE/$key.last" +window=$(seconds "$every") +if [ -f "$stamp" ] && [ $(( $(date +%s) - $(stat -c %Y "$stamp") )) -lt "$window" ]; then + log "skip [$key] (sent within $every): $msg" + exit 0 +fi + +if [ "$dry" = 1 ]; then echo "would alert ($severity): $msg"; exit 0; fi + +apikey=${SMSHUB_ALERT_API_KEY:-} +if [ -z "$apikey" ]; then + tmp=$(mktemp -d "$STATE/v.XXXXXX") + if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then + apikey=$(python3 -I -c 'import csv,sys +for r in csv.DictReader(open(sys.argv[1])): + if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv") + fi + rm -rf "$tmp" +fi + +body=$(jq -nc --arg m "$msg" --arg s "$severity" --arg t "$to" '{message:$m, severity:$s} + (if $t=="" then {} else {to:$t} end)') +out= code=000 +if [ -n "$apikey" ]; then + out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST https://smshub.dev/api/v1/alerts \ + -H "X-API-Key: $apikey" -H 'Content-Type: application/json' --data "$body" 2>&1) + code=${out##*$'\n'} + out=${out%$'\n'*} +fi + +if [ "$code" = 200 ] || [ "$code" = 201 ]; then + touch "$stamp" + log "sent [$key] $severity: $msg -> $(jq -r '(.alert.channels|join("+")) + ": " + .note' <<<"$out" 2>/dev/null || echo "$out")" + exit 0 +fi + +# smshub said no (or is down): email, so the failure still reaches a person. +reason=${out:-no API key (vault smshub--prod SMSHUB_ALERT_API_KEY)} +log "smshub failed [$key] HTTP $code: $reason" +if mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "[alert] $msg" --via resend \ + --body "$msg + +(smshub alert failed, HTTP $code: ${reason:0:300})" >/dev/null 2>&1; then + touch "$stamp" + log "emailed [$key] instead: $msg" + exit 0 +fi +log "FAILED [$key] nothing went out: $msg" +exit 1 diff --git a/bin/anthropic-credit-check b/bin/anthropic-credit-check new file mode 100755 index 0000000..6d7b467 --- /dev/null +++ b/bin/anthropic-credit-check @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""anthropic-credit-check: is the fleet's shared Anthropic key still able to answer? + +One key (vault profullstack-sharable-keys--prod, ANTHROPIC_API_KEY) is shared by +about 14 production apps, and each of them falls back silently when it stops +working. This asks it for one token. When the answer is "credit balance is too +low", a usage limit or an invalid key, it runs `alert` (at most once a day); +when it works again it clears that, so the next outage alerts at once. + + anthropic-credit-check check, alert on failure + anthropic-credit-check --dry-run check and print, never alert + +Exit 0 when the key works, 1 when it does not, 2 when the check itself could +not run (vault or network); only a definite answer from Anthropic alerts. +Cron: hourly. Log: ~/.local/state/alert/anthropic-credit.log +""" +import csv, json, os, subprocess, sys, tempfile, urllib.error, urllib.request +from datetime import datetime, timezone + +HOME = os.path.expanduser("~") +STATE = f"{HOME}/.local/state/alert" +KEY_NAME = "anthropic-credit" +MODEL = "claude-haiku-5-5" +DRY = "--dry-run" in sys.argv +os.environ["PATH"] = f"{HOME}/.local/bin:{HOME}/.local/share/mise/shims:/usr/local/bin:/usr/bin:/bin" +os.makedirs(STATE, exist_ok=True) + + +def log(msg): + line = f"{datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')} {msg}" + with open(f"{STATE}/anthropic-credit.log", "a") as f: + f.write(line + "\n") + if DRY or sys.stdout.isatty(): + print(line) + + +def vault_key(): + with tempfile.TemporaryDirectory(dir=STATE) as tmp: + out = f"{tmp}/v.csv" + subprocess.run(["logicsrc", "teams", "export", "profullstack", "profullstack-sharable-keys", "prod", + "-o", out, "--yes"], capture_output=True, timeout=90, check=True) + with open(out) as f: + for row in csv.DictReader(f): + if row["key"] == "ANTHROPIC_API_KEY": + return row["value"] + return None + + +def ask(key): + req = urllib.request.Request( + "https://api.anthropic.com/v1/messages", method="POST", + data=json.dumps({"model": MODEL, "max_tokens": 1, "messages": [{"role": "user", "content": "ok"}]}).encode(), + headers={"x-api-key": key, "anthropic-version": "2023-06-01", "content-type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return r.status, "" + except urllib.error.HTTPError as e: + try: + msg = json.loads(e.read()).get("error", {}).get("message", "") + except Exception: + msg = "" + return e.code, msg + + +def main(): + try: + key = vault_key() + except Exception as e: + log(f"could not read the key from the vault: {e}") + return 2 + if not key: + log("no ANTHROPIC_API_KEY in vault profullstack-sharable-keys--prod") + return 2 + try: + status, msg = ask(key) + except Exception as e: + log(f"could not reach Anthropic: {e}") + return 2 + + if status == 200: + log("ok") + subprocess.run(["alert", "--clear", KEY_NAME]) + return 0 + low = msg.lower() + broken = status in (401, 403) or "credit balance" in low or "usage limit" in low or "billing" in low + if not broken: + # 429 / 5xx / a model name problem: not the outage this watches for. + log(f"HTTP {status} (not a credit or key problem, no alert): {msg[:200]}") + return 2 + why = "is out of credits" if "credit balance" in low else "hit its usage limit" if "usage limit" in low else f"is refused (HTTP {status})" + text = f"The shared Anthropic API key {why}. About 14 apps have no AI until it is fixed." + log(f"HTTP {status}: {msg[:200]}") + if not DRY: + subprocess.run(["alert", "--key", KEY_NAME, "--every", "24h", text]) + else: + print(f"would alert: {text}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/fleet-nightly b/bin/fleet-nightly index d17e361..46e34e0 100755 --- a/bin/fleet-nightly +++ b/bin/fleet-nightly @@ -653,6 +653,10 @@ main().catch(async (e) => { const key = process.env.RESEND_API_KEY || loadSecrets().RESEND_API_KEY; if (key) await resend(key, { from: FROM, to: TO, subject: 'Fleet nightly FAILED', text: `The nightly fleet report for ${DAY} could not be produced.\n\n${msg}\n\nHost: ${hostname()}` }); } catch (e2) { console.error(`fleet-nightly: could not mail the failure either: ${e2.message}`); } + // And a text or call (bin/alert), since a failure email can fail the same way. + try { + execFileSync('alert', ['--key', 'fleet-nightly', '--every', '12h', `fleet-nightly failed for ${DAY}: ${String(e && e.message || e).slice(0, 200)}`], { stdio: 'ignore', timeout: 120_000 }); + } catch { console.error('fleet-nightly: alert failed too'); } } process.exit(1); }); diff --git a/bin/scripts-check b/bin/scripts-check index 4a375f6..1b6defb 100755 --- a/bin/scripts-check +++ b/bin/scripts-check @@ -65,6 +65,7 @@ for f in bin/* install.sh; do case "$(head -1 "$f")" in *bash*) bash -n "$f" 2>&1 | sed "s/^/ /" ; [ "${PIPESTATUS[0]}" = 0 ] || fail "$f does not parse" ;; *node*) node --check "$f" >/dev/null 2>&1 || fail "$f does not parse" ;; + *python*) python3 -I -c 'import ast,sys; ast.parse(open(sys.argv[1]).read(), sys.argv[1])' "$f" 2>&1 | sed "s/^/ /" ; [ "${PIPESTATUS[0]}" = 0 ] || fail "$f does not parse" ;; *sh*) sh -n "$f" 2>&1 | sed "s/^/ /" ; [ "${PIPESTATUS[0]}" = 0 ] || fail "$f does not parse" ;; *) note " no shebang, skipped: $f" ;; esac