diff --git a/README.md b/README.md index 9f5ac7a..8a6bc96 100644 --- a/README.md +++ b/README.md @@ -391,3 +391,25 @@ nichedb-nightly --from-json FILE render saved numbers instead of reading ``` Runs from cron at 05:05 UTC. A failure is mailed as "NicheDB nightly FAILED". + +## pairux-nightly + +The nightly PairUX report: installs, downloads and real users, in the same +shape as tron-nightly. PairUX has no telemetry and the desktop app does not +phone home for updates, so installs are counted where people fetch the bits: +outside IPs fetching `install.sh` / `install.ps1` and visiting `/download` in +dev2's shared nginx log (kept ~2 days, so the nightly must capture each day), +the daily rise in GitHub release installer downloads, Chocolatey, AUR, npm, +and GitHub clone counts of the brew/scoop/apt/rpm/nix/gentoo repos. Accounts +and usage (signups, meetings, joins, chat, recordings, channels, orgs) come +from the pairux Supabase Postgres on dev2, read-only, with our own and QA +accounts counted apart. + +``` +pairux-nightly mail yesterday's (UTC) report +pairux-nightly --dry-run build it under ~/.local/share/pairux-nightly, send nothing +pairux-nightly --day 2026-10-16 a specific UTC day +pairux-nightly --to a@b.com,c@d someone else (default anthony@ + preshy@profullstack.com) +``` + +Runs from cron at 05:45 UTC. A failure is mailed as "PairUX nightly FAILED". diff --git a/bin/pairux-nightly b/bin/pairux-nightly new file mode 100755 index 0000000..3210acc --- /dev/null +++ b/bin/pairux-nightly @@ -0,0 +1,534 @@ +#!/usr/bin/env node +// pairux-nightly — one email a night: how many people downloaded, installed and +// actually used PairUX yesterday, with our own accounts and boxes taken out. +// +// pairux-nightly send the report for yesterday (UTC) +// pairux-nightly --dry-run build it, write it under the data dir, send nothing +// pairux-nightly --day 2026-10-16 report a specific UTC day (installs need the +// nginx logs, which dev2 keeps for ~2 days) +// pairux-nightly --to a@b.com,c@d someone else (default anthony@ + preshy@profullstack.com) +// +// The desktop app has no telemetry and no auto-update ping, so "installs" can +// only be counted where people fetch the bits: +// script GET install.sh / install.ps1 on pairux.com, www and installer.pairux.com +// in dev2's shared nginx log: unique client IPs that are not our boxes +// page GET /download on pairux.com (people then click through to GitHub) +// github release asset download_count as a daily delta of the running total, +// app installers only (exe, dmg, mac zip, AppImage, deb, rpm). The ffmpeg +// assets are fetched by the install scripts, so their delta is a second +// count of script installs that got that far; SHA256SUMS.txt is fetched +// by the /download page in the browser and is left out. +// others Chocolatey downloads, AUR votes, npm downloads of the embed SDK, and +// GitHub clone counts of the brew/scoop/apt/rpm/nix/gentoo repos (which +// include our own release CI pushing to them). apt and rpm are served +// by GitHub Pages and winget keeps no counts, so those are n/a. +// Accounts and usage come from the pairux Supabase Postgres on dev2 (auth.users, +// sessions, session_participants, chat, recordings, channels, orgs). Internal +// accounts (ours, QA, example.com, mailinator) are counted apart. +// +// Every source may fail on its own and is named in the email. Secrets: only +// RESEND_API_KEY, from the team vault fleet-nightly--prod like fleet-nightly. +// The DB URL is the dev2 fleet kit state (~/.local/state/dev2-fleet/pairux.com.json). + +import { execFile, execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, readdirSync, writeFileSync, existsSync, rmSync, mkdtempSync } from 'node:fs'; +import { homedir, hostname, tmpdir } from 'node:os'; +import { join } from 'node:path'; + +process.env.PATH = [ + join(homedir(), '.local/bin'), + join(homedir(), '.local/share/mise/shims'), + join(homedir(), '.bun/bin'), + '/usr/local/bin', '/usr/bin', '/bin', + process.env.PATH || '', +].join(':'); + +const DATA_DIR = process.env.PAIRUX_NIGHTLY_DATA || join(homedir(), '.local/share/pairux-nightly'); +const FLEET_DAYS = join(homedir(), '.local/share/fleet-nightly/days'); +const STATE = join(process.env.DEV2_FLEET_STATE || join(homedir(), '.local/state/dev2-fleet'), 'pairux.com.json'); +const REPO = 'profullstack/pairux.com'; +const CHANNEL_REPOS = ['homebrew-pairux', 'scoop-pairux', 'pairux-apt', 'pairux-rpm', 'pairux-nix', 'gentoo-pairux']; +const NPM_PACKAGES = ['@profullstack/pairux-embed']; +const DEFAULT_FROM = 'PairUX nightly '; +const DEFAULT_TO = 'anthony@profullstack.com,preshy@profullstack.com'; +// Our own egress: dev1 (v4 + v6), dev2, and docker/wireguard ranges. +const OUR_IPS = (process.env.PAIRUX_OUR_IPS || '67.205.189.229,2604:a880:400:d1:0:4:c3fe:1,23.95.228.174').split(','); +const OURS = (ip) => OUR_IPS.includes(ip) || /^(10\.|127\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.)/.test(ip); +// Accounts that are us or our tests. +const INTERNAL = /@(profullstack\.com|pairux\.com|chovy\.com|example\.com|mailinator\.com)$|^ettinger@gmail\.com$|^qa[.+-]|\+qa@/i; +const STRIP_DAYS = 7; + +const args = process.argv.slice(2); +const flag = (name) => args.includes(name); +const opt = (name) => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; }; +const DRY = flag('--dry-run'); +const TO = (opt('--to') || process.env.PAIRUX_REPORT_TO || DEFAULT_TO).split(',').map((s) => s.trim()).filter(Boolean); +const FROM = process.env.PAIRUX_REPORT_FROM || DEFAULT_FROM; + +const isoDay = (d) => d.toISOString().slice(0, 10); +const addDays = (iso, k) => isoDay(new Date(Date.parse(`${iso}T00:00:00Z`) + k * 86_400_000)); +const dayLabel = (iso, withYear = false) => { + const d = new Date(`${iso}T00:00:00Z`); + const part = (o) => d.toLocaleDateString('en-US', { timeZone: 'UTC', ...o }); + return `${part({ weekday: 'short' })} ${part({ month: 'short' })} ${part({ day: 'numeric' })}${withYear ? `, ${part({ year: 'numeric' })}` : ''}`; +}; +const DAY = opt('--day') || addDays(isoDay(new Date()), -1); +if (!/^\d{4}-\d{2}-\d{2}$/.test(DAY)) throw new Error(`--day wants YYYY-MM-DD, got ${DAY}`); +const PREV = addDays(DAY, -1); +const DAYS = Array.from({ length: STRIP_DAYS }, (_, i) => addDays(DAY, i - STRIP_DAYS + 1)); + +const run = (cmd, argv, opts = {}) => new Promise((resolve, reject) => { + const p = execFile(cmd, argv, { maxBuffer: 64 << 20, timeout: opts.timeout || 120_000, env: { ...process.env, ...opts.env } }, (err, out, errOut) => + err ? reject(new Error(`${cmd}: ${(errOut || err.message).trim().split('\n').slice(-3).join(' ')}`)) : resolve(out)); + if (opts.input) { p.stdin.end(opts.input); } +}); + +function parseEnv(text) { + const out = {}; + for (const line of text.split('\n')) { + const m = /^\s*(?:export\s+)?([A-Z0-9_]+)\s*=\s*(.*)\s*$/.exec(line); + if (!m) continue; + let v = m[2]; + if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1); + out[m[1]] = v; + } + return out; +} + +function resendKey() { + if (process.env.RESEND_API_KEY) return process.env.RESEND_API_KEY; + const dir = mkdtempSync(join(tmpdir(), 'pairux-nightly-')); + try { + execFileSync('logicsrc', ['teams', 'pull', 'profullstack', 'fleet-nightly', 'prod', '--env', join(dir, 'env')], { stdio: 'ignore', timeout: 60_000 }); + const k = parseEnv(readFileSync(join(dir, 'env'), 'utf8')).RESEND_API_KEY; + if (k) return k; + } catch {} finally { rmSync(dir, { recursive: true, force: true }); } + const f = join(homedir(), '.config/logicsrc/shell.env'); + return existsSync(f) ? parseEnv(readFileSync(f, 'utf8')).RESEND_API_KEY : undefined; +} + +// ---------------------------------------------------------------- sources + +/** Accounts and usage, read-only, through Bun's Postgres client. Emails never leave this function. */ +async function accounts() { + const url = JSON.parse(readFileSync(STATE, 'utf8')).sb_database_url; + const script = ` + import { SQL } from "bun"; + const sql = new SQL(process.env.DBURL); + await sql.unsafe("set session characteristics as transaction read only"); + const [ro] = await sql.unsafe("show transaction_read_only"); + if (ro.transaction_read_only !== "on") throw new Error("not read-only"); + const q = (s) => sql.unsafe(s); + const out = { + users: await q("select id, email, created_at, email_confirmed_at is not null confirmed from auth.users order by created_at"), + sessions: await q("select id, host_user_id host, created_at, mode::text mode from sessions"), + parts: await q("select session_id s, user_id u, kind::text kind, joined_at from session_participants"), + chat: await q("select user_id u, created_at::date::text d, count(*)::int n from chat_messages group by 1, 2"), + scheduled: await q("select host_user_id host, created_at from scheduled_sessions"), + recordings: await q("select creator_id u, created_at, coalesce(duration_seconds, 0)::int secs from recordings"), + channels: await q("select owner_id u, created_at from channels"), + orgs: await q("select owner_id u, created_at from organizations"), + bookings: await q("select host_user_id u, created_at from booking_pages"), + analyses: await q("select host_user_id u, created_at from call_analyses"), + translation: await q("select user_id u, created_at from translation_usage"), + paid: await q("select (select count(*)::int from profiles where plan is not null and plan <> 'free') plans, (select count(*)::int from plan_payments where status in ('paid','confirmed','completed','credited')) payments"), + }; + console.log(JSON.stringify(out)); + await sql.close();`; + const out = await run('bun', ['-e', script], { env: { DBURL: url }, timeout: 90_000 }); + const db = JSON.parse(out.trim().split('\n').pop()); + const dayOf = (t) => (t ? new Date(t).toISOString().slice(0, 10) : null); + + const internalIds = new Set(db.users.filter((u) => INTERNAL.test(u.email || '')).map((u) => u.id)); + const ext = db.users.filter((u) => !internalIds.has(u.id)).map((u) => ({ ...u, created: dayOf(u.created_at) })); + const isExt = (id) => id && !internalIds.has(id); + const sessions = db.sessions.filter((s) => isExt(s.host)).map((s) => ({ ...s, day: dayOf(s.created_at) })); + const sessionIds = new Set(sessions.map((s) => s.id)); + const parts = db.parts.map((p) => ({ ...p, day: dayOf(p.joined_at) })); + + // "Used it" = hosted a meeting, joined one, or chatted in one. + const touch = {}; // user id -> set of days + const mark = (u, d) => { if (isExt(u) && d) (touch[u] ||= new Set()).add(d); }; + for (const s of sessions) mark(s.host, s.day); + for (const p of parts) mark(p.u, p.day); + for (const c of db.chat) mark(c.u, c.d); + + const within = (d, from, to) => d && d >= from && d <= to; + const signups = (from, to = DAY) => ext.filter((u) => within(u.created, from, to)).length; + const per = (d) => { + const joined = parts.filter((p) => p.day === d && sessionIds.has(p.s)); + const bySession = {}; + for (const p of joined) bySession[p.s] = (bySession[p.s] || 0) + 1; + return { + signups: ext.filter((u) => u.created === d).length, + confirmed: ext.filter((u) => u.created === d && u.confirmed).length, + active: Object.values(touch).filter((s) => s.has(d)).length, + meetings: sessions.filter((s) => s.day === d).length, + together: Object.values(bySession).filter((k) => k >= 2).length, + joins: joined.length, + guests: joined.filter((p) => !p.u).length, + }; + }; + const count = (rows, d, col = 'created_at') => rows.filter((r) => isExt(r.u ?? r.host) && dayOf(r[col]) === d).length; + const allExt = (rows) => rows.filter((r) => isExt(r.u ?? r.host)).length; + const usedEver = ext.filter((u) => touch[u.id]).length; + const day30 = addDays(DAY, -29); + + return { + total: db.users.length, + internal: internalIds.size, + external: ext.length, + confirmed: ext.filter((u) => u.confirmed).length, + new7: signups(addDays(DAY, -6)), + new30: signups(day30), + usedEver, + // Came back: used it on some day other than the one they signed up. + returned: ext.filter((u) => touch[u.id] && [...touch[u.id]].some((d) => d !== u.created)).length, + active30: Object.values(touch).filter((s) => [...s].some((d) => within(d, day30, DAY))).length, + lastSignup: ext.filter((u) => u.created <= DAY).pop()?.created ?? null, + recent: ext.filter((u) => u.created <= DAY).slice(-8).reverse().map((u) => ({ + domain: (u.email || '?@?').split('@')[1], created: u.created, confirmed: u.confirmed, used: !!touch[u.id], + })), + days: Object.fromEntries([PREV, ...DAYS].map((d) => [d, per(d)])), + meetingsAll: sessions.length, + meetingsSfu: sessions.filter((s) => s.mode === 'sfu').length, + agentJoins: parts.filter((p) => p.kind !== 'human').length, + agentJoinsDay: parts.filter((p) => p.kind !== 'human' && p.day === DAY).length, + chatDay: db.chat.filter((c) => isExt(c.u) && c.d === DAY).reduce((a, c) => a + c.n, 0), + scheduled: allExt(db.scheduled), scheduledDay: count(db.scheduled, DAY), + recordings: allExt(db.recordings), recordingsDay: count(db.recordings, DAY), + recordedHours: db.recordings.filter((r) => isExt(r.u)).reduce((a, r) => a + r.secs, 0) / 3600, + channels: allExt(db.channels), channelsDay: count(db.channels, DAY), + orgs: allExt(db.orgs), orgsDay: count(db.orgs, DAY), + bookings: allExt(db.bookings), + analyses: allExt(db.analyses), analysesDay: count(db.analyses, DAY), + translation: allExt(db.translation), translationDay: count(db.translation, DAY), + paidPlans: db.paid[0].plans, payments: db.paid[0].payments, + }; +} + +/** Installer fetches and /download views for one UTC day from dev2's shared nginx log; IPs leave only as hashes. */ +async function nginx() { + const [y, , d] = DAY.split('-'); + const mon = new Date(`${DAY}T00:00:00Z`).toLocaleDateString('en-US', { month: 'short', timeZone: 'UTC' }); + const stamp = `${d}/${mon}/${y}`; + // kind: sh / ps1 = install script, page = /download, site = installer.pairux.com landing page. + const remote = `cd /var/log/nginx && { cat access.log access.log.1 2>/dev/null; zcat access.log.*.gz 2>/dev/null; } | ` + + `awk -v s='[${stamp}:' 'index($0, s) && ($1=="pairux.com" || $1=="www.pairux.com" || $1=="installer.pairux.com") { ` + + `bot = tolower($0) ~ /bot|crawl|spider|slurp|preview|gatus|uptime|monitor|headless|python-requests|go-http/; k = ""; ` + + `if ($0 ~ /"(GET|HEAD) \\/install\\.sh/) k = "sh"; else if ($0 ~ /"(GET|HEAD) \\/install\\.ps1?[ ?]/) k = "ps1"; ` + + `else if (!bot && $1 != "installer.pairux.com" && $0 ~ /"GET \\/download[ ?]/) k = "page"; ` + + `else if (!bot && $1 == "installer.pairux.com" && $0 ~ /"GET \\/ /) k = "site"; ` + + `if (k != "") print k, $2 }' | sort | uniq -c; ` + + `echo "OLDEST $(ls -tr access.log.*.gz | head -1 | xargs zcat | head -1 | grep -oE '\\[[^]]+\\]')"`; + const local = hostname().startsWith('dev2'); + const out = local + ? await run('bash', ['-s'], { input: remote, timeout: 300_000 }) + : await run('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=15', 'dev2', 'bash', '-s'], { input: remote, timeout: 300_000 }); + let oldest = null; const rows = []; + for (const line of out.split('\n')) { + if (line.startsWith('OLDEST ')) { oldest = line.slice(7); continue; } + const m = /^\s*(\d+)\s+(\S+)\s+(\S+)$/.exec(line); + if (m) rows.push({ kind: m[2], ip: m[3], hits: +m[1] }); + } + const outside = rows.filter((x) => !OURS(x.ip)); + const ips = (kinds) => [...new Set(outside.filter((x) => kinds.includes(x.kind)).map((x) => x.ip))]; + const installers = ips(['sh', 'ps1']); + // A log that starts after the day began undercounts; say so instead of guessing. + const t = /\[(\d+)\/(\w+)\/(\d+):(\d+:\d+:\d+)/.exec(oldest || ''); + const complete = !t || Date.parse(`${t[1]} ${t[2]} ${t[3]} ${t[4]} UTC`) <= Date.parse(`${DAY}T00:00:00Z`); + return { + unique: installers.length, + sh: ips(['sh']).length, + ps1: ips(['ps1']).length, + fetches: outside.filter((x) => x.kind === 'sh' || x.kind === 'ps1').reduce((a, x) => a + x.hits, 0), + page: ips(['page']).length, + site: ips(['site']).length, + ours: new Set(rows.filter((x) => OURS(x.ip)).map((x) => x.ip)).size, + hashes: installers.map((ip) => createHash('sha256').update(`pairux-nightly:${ip}`).digest('hex').slice(0, 16)), + complete, oldest, + }; +} + +/** Release download totals; the nightly number is today's total minus the last snapshot's. */ +async function github() { + const out = await run('gh', ['api', `repos/${REPO}/releases?per_page=100`, '--paginate', '-q', + '.[] | {tag: .tag_name, published: .published_at, draft: .draft, pre: .prerelease, assets: [.assets[] | {name, n: .download_count}]}']); + const rels = out.trim().split('\n').filter(Boolean).map((l) => JSON.parse(l)).filter((r) => !r.draft); + const kind = (name) => /^ffmpeg-/.test(name) ? 'ffmpeg' : /\.(exe|dmg|zip|AppImage|deb|rpm)$/i.test(name) ? 'app' : 'other'; + const platform = (name) => /\.exe$/i.test(name) ? 'Windows' : /\.dmg$|mac\.zip$/i.test(name) ? 'macOS' : 'Linux'; + const byPlatform = {}; + let total = 0, ffmpeg = 0; + for (const r of rels) for (const a of r.assets) { + const k = kind(a.name); + if (k === 'app') { total += a.n; byPlatform[platform(a.name)] = (byPlatform[platform(a.name)] || 0) + a.n; } + else if (k === 'ffmpeg') ffmpeg += a.n; + } + const appN = (r) => r.assets.filter((a) => kind(a.name) === 'app').reduce((x, a) => x + a.n, 0); + const latest = rels.find((r) => !r.pre) || rels[0]; + const repo = JSON.parse(await run('gh', ['api', `repos/${REPO}`, '-q', '{stars: .stargazers_count, forks: .forks_count}'])); + return { total, ffmpeg, byPlatform, releases: rels.length, latest: latest && { tag: latest.tag, published: latest.published?.slice(0, 10), n: appN(latest) }, ...repo }; +} + +async function packages() { + const res = {}; + const job = async (name, fn) => { try { await fn(); } catch (e) { res[`${name}Error`] = e.message; } }; + await Promise.all([ + job('choco', async () => { + const x = await (await fetch(`https://community.chocolatey.org/api/v2/Packages()?$filter=Id%20eq%20'pairux'%20and%20IsLatestVersion`)).text(); + const g = (k) => (new RegExp(`]*>([^<]*)`).exec(x) || [])[1]; + if (g('Version')) res.choco = { version: g('Version'), total: +g('DownloadCount') || 0, version_n: +g('VersionDownloadCount') || 0 }; + }), + job('aur', async () => { + const j = await (await fetch('https://aur.archlinux.org/rpc/v5/info?arg[]=pairux-bin')).json(); + const r = j.results?.[0]; + if (r) res.aur = { version: r.Version, votes: r.NumVotes, popularity: r.Popularity }; + }), + job('npm', async () => { + res.npm = {}; + for (const p of NPM_PACKAGES) { + const j = await (await fetch(`https://api.npmjs.org/downloads/point/${DAY}/${p}`)).json(); + const all = await (await fetch(`https://api.npmjs.org/downloads/point/2020-01-01:${DAY}/${p}`)).json(); + res.npm[p] = { day: j.downloads ?? null, total: all.downloads ?? null }; + } + }), + // GitHub keeps 14 days of per-day clone counts; brew/scoop/gentoo/nix users clone or fetch these. + job('clones', async () => { + res.clones = {}; + for (const r of CHANNEL_REPOS) { + const j = JSON.parse(await run('gh', ['api', `repos/profullstack/${r}/traffic/clones`])); + const day = (j.clones || []).find((c) => c.timestamp.slice(0, 10) === DAY); + // GitHub publishes a day's traffic late; a missing day is unknown, not zero. + res.clones[r] = { count: day?.count ?? null, uniques: day?.uniques ?? null, count14: j.count, uniques14: j.uniques }; + } + }), + ]); + return res; +} + +/** pairux.com page views, from what fleet-nightly already wrote at 05:15. */ +function views() { + for (const f of [`${DAY}.json`, ...(existsSync(FLEET_DAYS) ? readdirSync(FLEET_DAYS).sort().reverse() : [])]) { + const p = join(FLEET_DAYS, f); + if (!existsSync(p)) continue; + const row = JSON.parse(readFileSync(p, 'utf8')).properties?.find((x) => x.host === 'pairux.com'); + if (row?.views && DAY in row.views) return row.views; + } + return null; +} + +// ---------------------------------------------------------------- history + +function history() { + const dir = join(DATA_DIR, 'days'); + const out = {}; + if (existsSync(dir)) for (const f of readdirSync(dir)) if (f.endsWith('.json')) { + try { out[f.slice(0, 10)] = JSON.parse(readFileSync(join(dir, f), 'utf8')); } catch {} + } + return out; +} + +// ---------------------------------------------------------------- render + +const INK = '#0b0b0b', MUTE = '#52514e', LINE = '#e6e5e1', UP = '#1a7f37', DOWN = '#b42318', BLUE = '#2a78d6'; +const n = (v) => (v === null || v === undefined ? '·' : Number(v).toLocaleString('en-US')); +const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c])); +const signed = (d) => (d > 0 ? `+${n(d)}` : d < 0 ? `-${n(-d)}` : '±0'); +const plural = (k, word) => `${n(k)} ${word}${k === 1 ? '' : 's'}`; + +function verdict(r) { + const a = r.accounts; + if (!a) return 'Accounts unreadable tonight.'; + return `${plural(a.external, 'account')} that are not us (of ${n(a.total)}; ${n(a.internal)} are ours or QA), ` + + `${n(a.new7)} new in 7 days and ${n(a.new30)} in 30. ` + + `${n(a.confirmed)} confirmed an email, ${n(a.usedEver)} ever hosted or joined a meeting, ${n(a.returned)} used it on a day after signup day. ` + + `Last outside signup: ${a.lastSignup ? dayLabel(a.lastSignup, true) : 'never'}.`; +} + +function renderHtml(r, meta) { + const tile = (label, cur, prev, sub) => ` +
${label}
+
${n(cur)}
+
${prev === null || cur === null ? ' ' : `${signed(cur - prev)} vs ${esc(dayLabel(PREV))}`}
+
${sub}
`; + + const a = r.accounts, g = r.github, c = r.nginx, p = r.packages; + const max = Math.max(1, ...DAYS.map((d) => (r.series[d]?.installs || 0) + (r.series[d]?.downloads || 0))); + const bars = DAYS.map((d) => { + const s = r.series[d] || {}; + const v = (s.installs ?? 0) + (s.downloads ?? 0); + const w = Math.max(v ? 2 : 0, Math.round((v / max) * 100)); + const known = s.installs !== undefined || s.downloads !== undefined; + const ad = a?.days[d]; + return `${esc(dayLabel(d))} +${w ? `` : ''}
  
+${known ? `${n(s.installs)} script · ${n(s.downloads)} gh` : 'no snapshot'}${ad ? ` · ${n(ad.signups)} signups · ${n(ad.meetings)} mtgs` : ''}`; + }).join(''); + + const row = (k, v) => `${k}${v}`; + const plat = g ? Object.entries(g.byPlatform).sort((x, y) => y[1] - x[1]).map(([k, v]) => `${k} ${n(v)}`).join(' · ') : ''; + const recent = a?.recent?.length ? a.recent.map((u) => `${esc(dayLabel(u.created))} · @${esc(u.domain)}${u.confirmed ? ' · confirmed' : ''}${u.used ? ' · used it' : ''}`).join('
') : 'none'; + const d = a?.days[DAY]; + const clones = p.clones ? Object.entries(p.clones).map(([k, v]) => `${esc(k.replace(/-?pairux-?/, '') || k)} ${n(v.uniques)}/${n(v.uniques14)}`).join(' · ') : null; + const npm = p.npm ? Object.entries(p.npm).map(([k, v]) => `${esc(k)} ${n(v.day)} (${n(v.total)} all-time)`).join('
') : null; + + return `
+
+
PairUX nightly · ${esc(dayLabel(DAY, true))}
+
UTC day · our own boxes and test accounts excluded · compared with ${esc(dayLabel(PREV))}
+
${esc(verdict(r))}
+ +${tile('Script installs', c ? c.unique : null, r.series[PREV]?.installs ?? null, c ? `unique outside IPs on install.sh/.ps1${c.complete ? '' : ' · log incomplete'}${r.newInstallers !== null ? ` · ${n(r.newInstallers)} never seen before` : ''}` : 'nginx log unreadable')} +${tile('App downloads', r.series[DAY]?.downloads ?? null, r.series[PREV]?.downloads ?? null, g ? `GitHub release installers · ${n(g.total)} all-time${r.series[DAY]?.downloads === undefined ? ' · first snapshot, delta from tomorrow' : ''}` : 'GitHub unreadable')} +${tile('New signups', d ? d.signups : null, a ? a.days[PREV].signups : null, a ? `${n(d.confirmed)} confirmed email · ${n(a.new7)} in 7d · ${n(a.new30)} in 30d` : 'DB unreadable')} +${tile('Used it', d ? d.active : null, a ? a.days[PREV].active : null, a ? 'outside users who hosted, joined or chatted' : '')} +
+ +
Last ${STRIP_DAYS} days
+${bars}
+ +
Meetings
+ +${a ? row('Meetings started that day', `${n(d.meetings)} by outside hosts · ${n(d.together)} with 2+ people joining · ${n(d.joins)} joins (${n(d.guests)} guests)`) : ''} +${a ? row('Meetings all-time', `${n(a.meetingsAll)} by outside hosts · ${n(a.meetingsSfu)} on the SFU`) : ''} +${a ? row('Agent participants', `${n(a.agentJoinsDay)} that day · ${n(a.agentJoins)} all-time`) : ''} +${a ? row('Chat messages', `${n(a.chatDay)} that day`) : ''} +${a ? row('Scheduled meetings', `${n(a.scheduledDay)} created that day · ${n(a.scheduled)} all-time · ${n(a.bookings)} booking pages`) : ''} +${a ? row('Recordings', `${n(a.recordingsDay)} that day · ${n(a.recordings)} all-time (${a.recordedHours.toFixed(1)} h)`) : ''} +${a ? row('Channels / orgs', `${n(a.channels)} channels (+${n(a.channelsDay)}) · ${n(a.orgs)} orgs (+${n(a.orgsDay)})`) : ''} +${a ? row('AI call analysis / translation', `${n(a.analyses)} analyses (+${n(a.analysesDay)}) · ${n(a.translation)} translation uses (+${n(a.translationDay)})`) : ''} +${a ? row('Paid', `${n(a.paidPlans)} paid plans · ${n(a.payments)} payments`) : ''} +${a ? row('Users active in the last 30 days', n(a.active30)) : ''} +${a ? row('Latest outside signups', recent) : ''} +
+ +
Downloads and installs
+ +${c ? row('Install scripts that day', `${n(c.sh)} install.sh · ${n(c.ps1)} install.ps1 · ${n(c.fetches)} fetches${c.ours ? ` · ${n(c.ours)} of our IPs left out` : ''}`) : ''} +${r.series[DAY]?.ffmpeg !== undefined ? row('ffmpeg fetched by the scripts', `${n(r.series[DAY].ffmpeg)} that day · ${n(g?.ffmpeg)} all-time`) : g ? row('ffmpeg fetched by the scripts', `${n(g.ffmpeg)} all-time (delta from tomorrow)`) : ''} +${c ? row('pairux.com/download', `${n(c.page)} unique outside visitors · installer.pairux.com page ${n(c.site)}`) : ''} +${g ? row('GitHub releases', `${n(g.releases)} releases · latest ${esc(g.latest?.tag || '?')} (${esc(g.latest?.published || '?')}) ${n(g.latest?.n)} downloads`) : ''} +${g ? row('Installers by platform, all-time', esc(plat)) : ''} +${g ? row('Repo', `${n(g.stars)} stars · ${n(g.forks)} forks`) : ''} +${row('Chocolatey', p.choco ? `${n(p.choco.total)} downloads all-time · ${esc(p.choco.version || '?')} ${n(p.choco.version_n)}` : 'n/a')} +${row('AUR pairux-bin', p.aur ? `${n(p.aur.votes)} votes · popularity ${esc(p.aur.popularity)}` : 'n/a')} +${row('brew / scoop / apt / rpm / nix / gentoo repo clones', clones ? `${clones}
unique cloners that day / 14 days; includes our release CI
` : 'n/a')} +${row('apt / rpm / winget installs', 'n/a (GitHub Pages and winget publish no counts)')} +${npm ? row('npm', npm) : ''} +${row('pairux.com views', r.views ? `${n(r.views[DAY])} humans (CrawlProof)` : 'not in fleet-nightly')} +
+ +${meta.problems.length ? `
Sources skipped: ${meta.problems.map(esc).join('; ')}
` : ''} +
PairUX sends no telemetry and the desktop app does not phone home for updates, so there is no count of running apps. Script installs = distinct client IPs that fetched install.sh or install.ps1 from pairux.com/installer.pairux.com that day, minus dev1/dev2/private ranges ("never seen" compares hashed IPs with every earlier night). App downloads = the day's rise in GitHub release installer download counts (exe, dmg, zip, AppImage, deb, rpm), which includes Homebrew, Scoop, Chocolatey and AUR installs, since they all pull from GitHub releases. "Used it" = an outside account hosted a meeting, joined one or sent chat that day. Accounts with ${esc(String(INTERNAL))} are counted as ours. Generated by pairux-nightly on ${esc(hostname())} at ${esc(meta.generatedAt)}.
+
`; +} + +function renderText(r, meta) { + const a = r.accounts, s = r.series[DAY] || {}, d = a?.days[DAY]; + return [ + `PairUX nightly · ${dayLabel(DAY, true)}`, + '', + verdict(r), + '', + `Script installs (unique outside IPs): ${n(r.nginx?.unique)}${r.nginx && !r.nginx.complete ? ' (log incomplete)' : ''}`, + `GitHub installer downloads: ${n(s.downloads)} (all-time ${n(r.github?.total)})`, + `pairux.com/download visitors: ${n(r.nginx?.page)}`, + `New outside signups: ${n(d?.signups)} (${n(d?.confirmed)} confirmed) · used it: ${n(d?.active)} · meetings: ${n(d?.meetings)} (${n(d?.together)} with 2+ people)`, + `pairux.com views: ${n(r.views?.[DAY])}`, + r.packages.choco ? `Chocolatey: ${n(r.packages.choco.total)} all-time` : '', + r.packages.aur ? `AUR: ${n(r.packages.aur.votes)} votes, popularity ${r.packages.aur.popularity}` : '', + meta.problems.length ? `\nSources skipped: ${meta.problems.join('; ')}` : '', + ].filter((l) => l !== '').join('\n'); +} + +function subjectLine(r) { + const a = r.accounts, s = r.series[DAY] || {}, d = a?.days[DAY]; + const parts = [ + r.nginx ? `${plural(r.nginx.unique, 'script install')}` : 'installs unknown', + s.downloads !== undefined ? `${n(s.downloads)} downloads` : null, + d ? plural(d.signups, 'signup') : null, + d ? plural(d.meetings, 'meeting') : null, + a ? `${n(a.external)} real users total` : null, + ].filter(Boolean); + return `PairUX ${dayLabel(DAY)}: ${parts.join(', ')}`; +} + +// ---------------------------------------------------------------- main + +async function resend(key, body) { + const r = await fetch('https://api.resend.com/emails', { + method: 'POST', + headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + const j = await r.json().catch(() => ({})); + if (!r.ok || !j.id) throw new Error(`Resend ${r.status}: ${JSON.stringify(j).slice(0, 300)}`); + return j.id; +} + +async function main() { + const t0 = Date.now(); + const problems = []; + const settle = async (name, fn) => { try { return await fn(); } catch (e) { problems.push(`${name} (${e.message.slice(0, 160)})`); return null; } }; + const [acc, ngx, gh, pk] = await Promise.all([ + settle('accounts', accounts), settle('nginx', nginx), settle('github', github), settle('packages', packages), + ]); + if (!acc && !ngx && !gh) throw new Error(`no source answered:\n ${problems.join('\n ')}`); + for (const [k, v] of Object.entries(pk || {})) if (k.endsWith('Error')) problems.push(`${k.slice(0, -5)} (${String(v).slice(0, 160)})`); + + const hist = history(); + // Downloads for DAY = this run's total minus the latest snapshot from before DAY. + const before = Object.keys(hist).filter((d) => d < DAY).sort().pop(); + const delta = (cur, key) => (cur !== undefined && before && hist[before][key] !== undefined ? Math.max(0, cur - hist[before][key]) : undefined); + const downloads = delta(gh?.total, 'githubTotal'); + const ffmpeg = delta(gh?.ffmpeg, 'ffmpegTotal'); + const seen = new Set(Object.entries(hist).filter(([d]) => d < DAY).flatMap(([, h]) => h.installerHashes || [])); + const newInstallers = ngx && Object.keys(hist).some((d) => d < DAY) ? ngx.hashes.filter((h) => !seen.has(h)).length : null; + const v = views(); + + const snap = { + day: DAY, + installs: ngx?.unique, installsComplete: ngx?.complete, installerHashes: ngx?.hashes, downloadPage: ngx?.page, + githubTotal: gh?.total, downloads, ffmpegTotal: gh?.ffmpeg, ffmpeg, + signups: acc?.days[DAY].signups, active: acc?.days[DAY].active, meetings: acc?.days[DAY].meetings, external: acc?.external, + views: v?.[DAY], + }; + const series = {}; + for (const d of DAYS) { + const h = d === DAY ? snap : hist[d]; + if (h) series[d] = { installs: h.installs, downloads: h.downloads, ffmpeg: h.ffmpeg, views: h.views ?? v?.[d] }; + else if (v && d in v) series[d] = { views: v[d] }; + } + const r = { accounts: acc, nginx: ngx, github: gh, packages: pk || {}, views: v, series, newInstallers }; + const meta = { generatedAt: new Date().toISOString().replace('T', ' ').slice(0, 16) + ' UTC', problems }; + const html = renderHtml(r, meta), text = renderText(r, meta), subject = subjectLine(r); + + mkdirSync(join(DATA_DIR, 'days'), { recursive: true }); + mkdirSync(join(DATA_DIR, 'out'), { recursive: true }); + writeFileSync(join(DATA_DIR, 'out', 'latest.html'), html); + writeFileSync(join(DATA_DIR, 'out', 'latest.txt'), text); + writeFileSync(join(DATA_DIR, 'days', `${DAY}.json`), JSON.stringify(snap)); + + if (DRY) { + console.log(text); + console.log(`\ndry run · subject: ${subject} · wrote ${join(DATA_DIR, 'out', 'latest.html')} · ${((Date.now() - t0) / 1000).toFixed(1)}s`); + return; + } + const key = resendKey(); + if (!key) throw new Error('no RESEND_API_KEY to send with'); + const id = await resend(key, { from: FROM, to: TO, subject, html, text }); + console.log(`sent ${id} to ${TO.join(', ')}: ${subject} (${((Date.now() - t0) / 1000).toFixed(1)}s)`); +} + +main().catch(async (e) => { + const msg = String(e && e.stack || e); + console.error(`pairux-nightly: ${msg}`); + if (!DRY) { + try { + const key = resendKey(); + if (key) await resend(key, { from: FROM, to: TO, subject: 'PairUX nightly FAILED', text: `The PairUX report for ${DAY} could not be produced.\n\n${msg}\n\nHost: ${hostname()}` }); + } catch (e2) { console.error(`pairux-nightly: could not mail the failure either: ${e2.message}`); } + } + process.exit(1); +});