fix(ci): Retry binary downloads on transient HTTP errors

apps/code/scripts/download-binaries.mjs

@@ -6,10 +6,12 @@ import {
   createWriteStream,
   existsSync,
   mkdirSync,
+  realpathSync,
   rmSync,
 } from "node:fs";
 import { dirname, join } from "node:path";
 import { pipeline } from "node:stream/promises";
+import { setTimeout as sleep } from "node:timers/promises";
 import { fileURLToPath } from "node:url";
 import { extract } from "tar";
 
@@ -75,14 +77,45 @@ const BINARIES = [
   },
 ];
 
-async function downloadFile(url, destPath) {
+export const MAX_DOWNLOAD_ATTEMPTS = 5;
+const RETRIABLE_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
+
+class NonRetriableError extends Error {}
+
+function backoffDelayMs(attempt) {
+  const base = Math.min(1000 * 2 ** (attempt - 1), 15000);
+  return Math.floor(base * (0.5 + Math.random() * 0.5));
+}
+
+export async function downloadFile(url, destPath) {
   console.log(`  Downloading: ${url}`);
-  const response = await fetch(url, { redirect: "follow" });
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  for (let attempt = 1; attempt <= MAX_DOWNLOAD_ATTEMPTS; attempt++) {
+    try {
+      const response = await fetch(url, { redirect: "follow" });
+      if (!response.ok) {
+        const message = `HTTP ${response.status}: ${response.statusText}`;
+        if (RETRIABLE_HTTP_STATUSES.has(response.status)) {
+          throw new Error(message);
+        }
+        throw new NonRetriableError(message);
+      }
+      await pipeline(response.body, createWriteStream(destPath));
+      console.log(`  Saved to: ${destPath}`);
+      return;
+    } catch (error) {
+      if (
+        error instanceof NonRetriableError ||
+        attempt === MAX_DOWNLOAD_ATTEMPTS
+      ) {
+        throw error;
+      }
+      const delayMs = backoffDelayMs(attempt);
+      console.warn(
+        `  Attempt ${attempt}/${MAX_DOWNLOAD_ATTEMPTS} failed: ${error.message}. Retrying in ${(delayMs / 1000).toFixed(1)}s...`,
+      );
+      await sleep(delayMs);
+    }
   }
-  await pipeline(response.body, createWriteStream(destPath));
-  console.log(`  Saved to: ${destPath}`);
 }
 
 async function extractArchive(archivePath, destDir) {
@@ -156,7 +189,12 @@ async function main() {
   console.log("\nDone.");
 }
 
-main().catch((err) => {
-  console.error("\nFailed:", err.message);
-  process.exit(1);
-});
+const isEntrypoint =
+  process.argv[1] &&
+  realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isEntrypoint) {
+  main().catch((err) => {
+    console.error("\nFailed:", err.message);
+    process.exit(1);
+  });
+}

apps/code/scripts/download-binaries.test.mjs

@@ -0,0 +1,113 @@
+import { setTimeout as sleep } from "node:timers/promises";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { downloadFile, MAX_DOWNLOAD_ATTEMPTS } from "./download-binaries.mjs";
+
+vi.mock("node:timers/promises", () => {
+  const setTimeout = vi.fn(() => Promise.resolve());
+  return { setTimeout, default: { setTimeout } };
+});
+vi.mock("node:stream/promises", () => {
+  const pipeline = vi.fn(() => Promise.resolve());
+  return { pipeline, default: { pipeline } };
+});
+vi.mock("node:fs", () => {
+  const fns = {
+    chmodSync: vi.fn(),
+    createWriteStream: vi.fn(() => ({})),
+    existsSync: vi.fn(() => true),
+    mkdirSync: vi.fn(),
+    realpathSync: vi.fn(() => "/not/the/entrypoint"),
+    rmSync: vi.fn(),
+  };
+  return { ...fns, default: fns };
+});
+
+const okResponse = () => ({
+  ok: true,
+  status: 200,
+  statusText: "OK",
+  body: {},
+});
+const errorResponse = (status, statusText) => ({
+  ok: false,
+  status,
+  statusText,
+  body: null,
+});
+
+describe("downloadFile", () => {
+  let fetchMock;
+
+  beforeEach(() => {
+    fetchMock = vi.fn();
+    vi.stubGlobal("fetch", fetchMock);
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+    vi.unstubAllGlobals();
+  });
+
+  it("downloads on the first attempt without retrying", async () => {
+    fetchMock.mockResolvedValue(okResponse());
+
+    await downloadFile("https://example.test/bin.tar.gz", "/tmp/bin.tar.gz");
+
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it("retries retriable HTTP statuses then succeeds", async () => {
+    fetchMock
+      .mockResolvedValueOnce(errorResponse(503, "Service Unavailable"))
+      .mockResolvedValueOnce(errorResponse(504, "Gateway Time-out"))
+      .mockResolvedValueOnce(okResponse());
+
+    await downloadFile("u", "/tmp/bin");
+
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it("fails fast on non-retriable HTTP statuses", async () => {
+    fetchMock.mockResolvedValue(errorResponse(404, "Not Found"));
+
+    await expect(downloadFile("u", "/tmp/bin")).rejects.toThrow("HTTP 404");
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it("retries network-level errors that carry no HTTP status", async () => {
+    fetchMock
+      .mockRejectedValueOnce(new Error("ECONNRESET"))
+      .mockRejectedValueOnce(new TypeError("fetch failed"))
+      .mockResolvedValueOnce(okResponse());
+
+    await downloadFile("u", "/tmp/bin");
+
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it("gives up after MAX_DOWNLOAD_ATTEMPTS and rethrows the last error", async () => {
+    fetchMock.mockResolvedValue(errorResponse(503, "Service Unavailable"));
+
+    await expect(downloadFile("u", "/tmp/bin")).rejects.toThrow("HTTP 503");
+    expect(fetchMock).toHaveBeenCalledTimes(MAX_DOWNLOAD_ATTEMPTS);
+    expect(sleep).toHaveBeenCalledTimes(MAX_DOWNLOAD_ATTEMPTS - 1);
+  });
+
+  it("backs off exponentially with jitter inside the expected bounds", async () => {
+    fetchMock.mockResolvedValue(errorResponse(503, "Service Unavailable"));
+
+    await expect(downloadFile("u", "/tmp/bin")).rejects.toThrow();
+
+    const delays = sleep.mock.calls.map(([ms]) => ms);
+    expect(delays).toHaveLength(MAX_DOWNLOAD_ATTEMPTS - 1);
+    delays.forEach((delay, i) => {
+      const base = Math.min(1000 * 2 ** i, 15000);
+      expect(delay).toBeGreaterThanOrEqual(base * 0.5);
+      expect(delay).toBeLessThan(base);
+    });
+  });
+});