Live progress reporting from Python → Worker → loading page

- Worker /api/progress (POST): accepts a {phase, message, ...counts}
  JSON body, auth'd by Bearer token == GH_DISPATCH_TOKEN. Caches
  per-user in Workers Cache (1h TTL).
- Worker /api/status: surfaces the latest /api/progress payload under
  a 'progress' field (alongside CI step list + rate-limit + log tail).
- generate_stats.py: new report_progress(phase, message, **counts)
  helper that POSTs to STATS_PROGRESS_URL; called at each major phase
  boundary so the loading page can show real-time updates.
- Action: passes STATS_PROGRESS_TOKEN env var to the mining step.
- Loading page: new phase-msg panel above the step list rendering
  the current phase + counts (repos, commits, prs, issues, stars,
  repos_accessible).

Author: pirate

.github/workflows/mine-and-deploy.yml

@@ -83,6 +83,9 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_MINING_TOKEN }}
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          # Posted by generate_stats.py to /api/progress so the Worker's
+          # loading page can render real-time phase info.
+          STATS_PROGRESS_TOKEN: ${{ secrets.GH_MINING_TOKEN }}
         run: |
           set -e
 

cloudflare/worker/index.ts

@@ -27,6 +27,9 @@ export default {
     if (url.pathname === "/api/status") {
       return handleStatus(req, env, url);
     }
+    if (url.pathname === "/api/progress") {
+      return handleProgress(req, env, url);
+    }
 
     // -- Static assets ----------------------------------------------------
     const assetResp = await env.ASSETS.fetch(req);
@@ -135,6 +138,46 @@ function json(obj: unknown, status = 200): Response {
 }
 
 
+async function handleProgress(
+  req: Request,
+  env: Env,
+  url: URL,
+): Promise<Response> {
+  // POST { phase, message, totals } — keyed by ?user=X — read back via
+  // /api/status. Auth: Bearer header must match GH_DISPATCH_TOKEN
+  // (the same token the Action gets from secrets) so random clients
+  // can't spam fake progress.
+  if (req.method !== "POST") {
+    return json({ error: "POST only" }, 405);
+  }
+  const auth = req.headers.get("Authorization") ?? "";
+  if (!auth.startsWith("Bearer ") ||
+      auth.slice(7) !== env.GH_DISPATCH_TOKEN) {
+    return json({ error: "unauthorized" }, 401);
+  }
+  const user = url.searchParams.get("user")?.trim();
+  if (!user || !VALID_LOGIN.test(user)) {
+    return json({ error: "invalid user" }, 400);
+  }
+  const body = await req.text();
+  // Sanity cap — small JSON only.
+  if (body.length > 4096) return json({ error: "too large" }, 413);
+  try { JSON.parse(body); } catch { return json({ error: "bad json" }, 400); }
+  const cache = caches.default;
+  await cache.put(
+    new Request(`https://internal-progress.invalid/${user}`),
+    new Response(body, {
+      headers: {
+        "Cache-Control": "max-age=3600",
+        "Content-Type": "application/json",
+        "X-Received-At": new Date().toISOString(),
+      },
+    }),
+  );
+  return json({ ok: true });
+}
+
+
 async function handleStatus(
   req: Request,
   env: Env,
@@ -215,6 +258,49 @@ async function handleStatus(
     }
   } catch {}
 
+  // Tail the job's live log for richer progress info (e.g. the Python
+  // script's `>> [N/M] ...` lines). The GH API redirects to a signed
+  // download URL — fetch() follows by default.
+  let recentLog: string[] = [];
+  if (job?.id) {
+    try {
+      const lr = await fetch(
+        `https://api.github.com/repos/${repo}/actions/jobs/${job.id}/logs`,
+        {
+          headers: {
+            Authorization: `Bearer ${env.GH_DISPATCH_TOKEN}`,
+            "User-Agent": "githubusers-archivebox-io",
+            Accept: "application/vnd.github+json",
+          },
+        },
+      );
+      if (lr.ok) {
+        const txt = await lr.text();
+        // Each line is "<ISO timestamp> <message>"; strip timestamp +
+        // filter to lines that look like Python script output.
+        const interesting = txt
+          .split("\n")
+          .map((l) => l.replace(/^\d{4}-\d{2}-\d{2}T[\d:.]+Z\s?/, ""))
+          .filter((l) => /^(>>|\s*\[|\s*-{2}|\s*!|\s*resolved\b|\s*scanning |\s*fetching |\s*mining |\s*deploying|\s*search quota|\s*resolving )/i
+                          .test(l))
+          .slice(-20);
+        recentLog = interesting;
+      }
+    } catch {}
+  }
+
+  // Read the latest progress update posted by the running Python script.
+  let progress: any = null;
+  try {
+    const pres = await caches.default.match(
+      new Request(`https://internal-progress.invalid/${user}`),
+    );
+    if (pres) {
+      progress = await pres.json();
+      progress.received_at = pres.headers.get("X-Received-At");
+    }
+  } catch {}
+
   return json({
     ok: true,
     run_id: run.id,
@@ -227,6 +313,8 @@ async function handleStatus(
                   ?? steps.at(-1)?.name ?? null,
     steps,
     rate_limit: rateLimit,
+    recent_log: recentLog,
+    progress,
   });
 }
 
@@ -323,6 +411,30 @@ function loadingPage(user: string): string {
     display: block; height: 100%; background: #3fb950;
   }
   .ratelimit.cooldown .gauge > span { background: #d97706; }
+  .phase-msg {
+    background: #0e2640; border: 1px solid #1f4d7a;
+    color: #58a6ff; padding: 12px 14px; border-radius: 6px;
+    margin: 0 0 14px; font-size: 13px;
+    display: flex; justify-content: space-between; align-items: center;
+    gap: 12px; flex-wrap: wrap;
+  }
+  .phase-msg .pm-msg { flex: 1; min-width: 200px; }
+  .phase-msg .pm-counts {
+    font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+    font-size: 11px; color: #8b949e;
+  }
+  .phase-msg .pm-counts strong { color: #c9d1d9; }
+  .livelog {
+    background: #0d1117; border: 1px solid #21262d;
+    border-radius: 6px; padding: 10px 12px; margin: 14px 0 0;
+    font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+    font-size: 11px; color: #c9d1d9;
+    max-height: 180px; overflow-y: auto; line-height: 1.45;
+    white-space: pre-wrap; word-break: break-all;
+  }
+  .livelog .l-hdr { color: #58a6ff; }
+  .livelog .l-warn { color: #ffa657; }
+  .livelog .l-err { color: #f85149; }
   a { color: #58a6ff; }
   code { background: #21262d; padding: 1px 5px; border-radius: 3px;
          font-size: 90%; font-family: inherit; }
@@ -348,10 +460,14 @@ function loadingPage(user: string): string {
 
     <div class="progress-track"><div class="progress-fill" id="progress"></div></div>
 
+    <div id="phase-msg" class="phase-msg" style="display:none"></div>
+
     <div id="ratelimit" class="ratelimit" style="display:none"></div>
 
     <ol class="steps" id="steps"></ol>
 
+    <pre id="livelog" class="livelog" style="display:none"></pre>
+
     <div id="error" class="err" style="display:none"></div>
 
     <div class="footer-row">
@@ -374,6 +490,8 @@ const $err = document.getElementById("error");
 const $runLink = document.getElementById("run-link");
 const $spinner = document.getElementById("hdr-spinner");
 const $rl = document.getElementById("ratelimit");
+const $log = document.getElementById("livelog");
+const $pmsg = document.getElementById("phase-msg");
 
 const startedAt = Date.now();
 function fmtElapsed(sec) {
@@ -440,6 +558,38 @@ async function checkDeployed() {
   }
 }
 
+function renderProgress(p) {
+  if (!p || !p.phase) { $pmsg.style.display = "none"; return; }
+  const countKeys = ["repos", "commits", "prs", "issues", "stars",
+                     "repos_accessible"];
+  const counts = countKeys
+    .filter(k => p[k] != null)
+    .map(k => '<strong>' + p[k] + '</strong> ' + k);
+  $pmsg.innerHTML =
+    '<div class="pm-msg">' +
+      (p.message || p.phase) +
+      ' <code style="font-size:10px;color:#8b949e;margin-left:6px">' +
+      p.phase + '</code></div>' +
+    (counts.length ? '<div class="pm-counts">' + counts.join(" · ") + '</div>' : "");
+  $pmsg.style.display = "flex";
+}
+
+function renderLog(lines) {
+  if (!Array.isArray(lines) || lines.length === 0) {
+    $log.style.display = "none"; return;
+  }
+  const esc = (s) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+  $log.innerHTML = lines.map(l => {
+    const cls = /^!|fail|error|❌/i.test(l) ? "l-err"
+              : /quota|warn|^  !/i.test(l) ? "l-warn"
+              : /^>>/.test(l) ? "l-hdr"
+              : "";
+    return '<span class="' + cls + '">' + esc(l) + '</span>';
+  }).join("\n");
+  $log.style.display = "block";
+  $log.scrollTop = $log.scrollHeight;
+}
+
 function renderRateLimit(rl) {
   if (!rl || (!rl.search && !rl.core)) {
     $rl.style.display = "none"; return;
@@ -521,7 +671,11 @@ function renderSteps(status) {
       checkDeployed(),
     ]);
     renderSteps(status);
-    if (status) renderRateLimit(status.rate_limit);
+    if (status) {
+      renderProgress(status.progress);
+      renderRateLimit(status.rate_limit);
+      renderLog(status.recent_log);
+    }
     if (deployed) {
       clearInterval(interval);
       $now.textContent = "Dashboard ready — reloading…";

generate_stats.py

@@ -43,6 +43,15 @@
 GH_LOGIN = "pirate"
 GH_NAME = "Nick Sweeting"
 
+# Optional: when set, the script POSTs phase updates to this URL so the
+# live "mining…" page can show real-time progress. The Worker's
+# /api/progress endpoint checks the Bearer token against GH_DISPATCH_TOKEN.
+PROGRESS_URL = os.environ.get(
+    "STATS_PROGRESS_URL",
+    "https://githubusers.archivebox.io/api/progress",
+)
+PROGRESS_TOKEN = os.environ.get("STATS_PROGRESS_TOKEN", "")
+
 # Known author emails pirate has used over the years.
 PIRATE_EMAILS = {
     "nikisweeting@gmail.com",
@@ -2364,6 +2373,35 @@ def _collect_all_records() -> list[dict]:
     return all_recs
 
 
+def report_progress(phase: str, message: str = "", **extra) -> None:
+    """POST a small JSON progress update to the live /api/progress endpoint
+    so the user's loading page can render real-time phase info. Best-effort
+    — failures are silent."""
+    if not PROGRESS_URL or not PROGRESS_TOKEN:
+        return
+    try:
+        import urllib.request, urllib.error
+        payload = {
+            "phase": phase,
+            "message": message,
+            "ts": time.time(),
+            "user": GH_LOGIN,
+            **extra,
+        }
+        data = json.dumps(payload).encode()
+        url = f"{PROGRESS_URL}?user={GH_LOGIN}"
+        req = urllib.request.Request(
+            url, data=data, method="POST",
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {PROGRESS_TOKEN}",
+            },
+        )
+        urllib.request.urlopen(req, timeout=5).read()
+    except Exception:
+        pass
+
+
 def _render_now(*, mining_status: str = "complete") -> dict:
     """Aggregate from cache and write stats.html. Returns aggregate dict.
     `mining_status` is embedded in the output so the live-mining UI knows
@@ -2539,6 +2577,8 @@ def main() -> int:
         print(f"Wrote {OUTPUT_FILE}")
         return 0
 
+    report_progress("starting", f"Mining @{GH_LOGIN}")
+
     # ---- Phase 1: local filesystem mining --------------------------------
     if not args.no_local:
         print(">> [1/7] Walking filesystem for local git repos ...", file=sys.stderr)
@@ -2551,8 +2591,11 @@ def main() -> int:
 
     # ---- Phase 2: bare-clone owned/org GH repos we don't have locally ----
     if not args.no_clone:
+        report_progress("phase-2-listing", "Listing accessible GitHub repos")
         print(">> [2/7] Listing accessible GitHub repos ...", file=sys.stderr)
         accessible = list_accessible_repos()
+        report_progress("phase-2-cloning", f"{len(accessible)} repos accessible — cloning",
+                        repos_accessible=len(accessible))
         print(f">> {len(accessible)} accessible repos via API", file=sys.stderr)
         # For generic --user runs, derive company patterns from the user's
         # top org owners so the dashboard still has some color coding.
@@ -2620,14 +2663,19 @@ def main() -> int:
     except Exception:
         pass
     if not args.no_prs:
+        report_progress("phase-4-prs", "Searching PRs + issues authored by user")
         print(">> [4/7] Searching PRs + issues by the user ...", file=sys.stderr)
         try:
             prs, issues = list_prs_and_issues()
+            report_progress("phase-4-prs-done",
+                            f"Found {len(prs)} PRs, {len(issues)} issues",
+                            prs=len(prs), issues=len(issues))
             print(f">> {len(prs)} PRs, {len(issues)} issues", file=sys.stderr)
         except Exception as e:
             print(f"  ! PR/issue search failed: {e}", file=sys.stderr)
 
         # ---- Phase 5: PR-detail fetch (lines added/removed) --------------
+        report_progress("phase-5-pr-details", "Fetching per-PR additions/deletions")
         print(">> [5/7] Fetching merged-PR additions/deletions ...",
               file=sys.stderr)
         try:
@@ -2637,6 +2685,8 @@ def main() -> int:
             print(f"  ! PR-detail fetch failed: {e}", file=sys.stderr)
 
         # ---- Phase 5a: per-PR commits (handles non-GH-linked author emails)
+        report_progress("phase-5a-pr-commits",
+                        "Walking each merged PR's commit list")
         try:
             existing_shas: set[str] = {r.get("sha")
                                        for r in _collect_all_records()
@@ -2697,6 +2747,8 @@ def main() -> int:
 
     # ---- Phase 6: star counts (full coverage) ----------------------------
     if not args.no_stars:
+        report_progress("phase-6-stars",
+                        "Fetching star counts + total-commits per repo")
         print(">> [6/7] Fetching star counts for all repos ...",
               file=sys.stderr)
         # Collect every canonical repo name in our data + every repo in
@@ -2743,6 +2795,9 @@ def main() -> int:
     print(f">> DONE: {t['commits']} commits, {t['repos']} repos, "
           f"{t['stars']:,}⭐ {t['prs_merged']}/{t['prs']} PRs merged, "
           f"{t['issues']} issues", file=sys.stderr)
+    report_progress("done", "Mining complete",
+                    commits=t['commits'], repos=t['repos'],
+                    stars=t['stars'], prs=t['prs'], issues=t['issues'])
     print(f"Wrote {OUTPUT_FILE}")
     return 0