fix: make nav-header check pass by reading llms-full.txt from disk (#2609)

The nav-header check was failing CI on the ~42MB `llms-full.txt`: the
server truncates the body mid-stream through the CI proxy, so no HTTP
client can read it cleanly (undici reports `terminated`, curl reports
`transfer closed with N bytes remaining`). The existing `assert_header`
only ever "passed" because it greps the header and ignores curl's exit
code.

The leak guard is a build-artifact invariant anyway - the nav header
must never be concatenated into `llms-full.txt` (issue #2557) - so it
now reads the file straight off `build/` instead of over HTTP. Per-page
checks stay over HTTP via curl (small, reliable). Verified locally: 14
pages + leak guard pass, and the guard still fires if the header is
present.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: marekh19

scripts/checkNavHeaders.mjs

@@ -7,13 +7,24 @@
 // served pages actually carry that header and that it never leaked into
 // `llms-full.txt`.
 //
-// It runs against a live server over HTTP (in CI: Nginx on :8080), so it tests
-// the real served bytes, not the files on disk.
+// The per-page checks run against a live server over HTTP (in CI: Nginx on
+// :8080), so they test the real served bytes. The llms-full.txt leak guard
+// instead reads the build artifact off disk - see readLlmsFull() for why.
 //
 // Usage:
 //   node scripts/checkNavHeaders.mjs [baseUrl]
 //   baseUrl defaults to $NAV_HEADERS_BASE_URL or http://localhost:8080
 
+import { execFile } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
+
+// Resolved relative to this script, so the disk read works regardless of cwd.
+const LLMS_FULL_FILE = fileURLToPath(new URL('../build/llms-full.txt', import.meta.url));
+
 const BASE = (process.argv[2] || process.env.NAV_HEADERS_BASE_URL || 'http://localhost:8080').replace(/\/$/, '');
 const SITE_URL = 'https://docs.apify.com';
 
@@ -58,20 +69,40 @@ const PAGES = [
 const failures = [];
 const fail = (page, msg) => failures.push(`${page} → ${msg}`);
 
-async function fetchText(url, attempts = 3) {
-    for (let i = 1; ; i++) {
-        try {
-            const res = await fetch(url, { headers: { Accept: 'text/markdown' } });
-            if (res.ok) return await res.text();
-            throw new Error(`HTTP ${res.status} for ${url}`);
-        } catch (err) {
-            // A 4xx is deterministic (e.g. a genuinely missing page) - fail fast.
-            // Anything else is transient: a 5xx, or undici throwing "terminated"
-            // when a pooled keep-alive socket goes stale mid-read (seen on the
-            // ~42MB llms-full.txt with the server's Keep-Alive: timeout=5). Retry.
-            if (i >= attempts || /^HTTP 4\d\d /.test(err.message)) throw err;
-            await new Promise((resolve) => setTimeout(resolve, 250 * i));
-        }
+// Fetch with curl rather than Node's global fetch (undici). undici throws
+// "terminated" reading the ~42MB llms-full.txt through the CI Nginx proxy, while
+// curl - which every other assertion in this CI job already uses - streams it
+// reliably. Flags: -f makes an HTTP >=400 a non-zero exit; -sS stays quiet but
+// still reports real errors; --retry covers genuinely transient blips.
+async function fetchText(url) {
+    try {
+        const { stdout } = await execFileAsync(
+            'curl',
+            ['-fsS', '--retry', '2', '--retry-delay', '1', '-H', 'Accept: text/markdown', url],
+            // llms-full.txt is ~42MB; the default 1MB maxBuffer would truncate it.
+            { maxBuffer: 256 * 1024 * 1024 },
+        );
+        return stdout;
+    } catch (err) {
+        throw new Error((err.stderr || err.message || '').toString().trim() || `curl failed for ${url}`);
+    }
+}
+
+// Read llms-full.txt for the leak guard. This is deliberately NOT an HTTP fetch:
+// the file is ~42MB and the CI server truncates the response mid-body (undici
+// reports "terminated", curl reports "transfer closed with N bytes remaining"),
+// so no client can read it reliably over the wire. The leak guard is anyway a
+// build-artifact invariant - the nav header must never be concatenated into
+// llms-full.txt (issue #2557), which is about what addNavHeaders.mjs leaves in
+// build/, not how it's served - so read the file straight off disk. Fall back to
+// HTTP only when there is no local build (e.g. checking a remote deployment).
+async function readLlmsFull() {
+    try {
+        return { source: LLMS_FULL_FILE, text: await readFile(LLMS_FULL_FILE, 'utf8') };
+    } catch (err) {
+        if (err.code !== 'ENOENT') throw err;
+        const url = `${BASE}/llms-full.txt`;
+        return { source: url, text: await fetchText(url) };
     }
 }
 
@@ -164,16 +195,16 @@ for (const { path, keys } of PAGES) {
 // synthetic root breadcrumb appears only in the header, so its presence here
 // means a leak.
 try {
-    const llmsFull = await fetchText(`${BASE}/llms-full.txt`);
-    if (llmsFull.includes(ROOT_PARENT)) {
-        fail('/llms-full.txt', 'nav header leaked into llms-full.txt (found the root breadcrumb)');
-        console.log('❌ /llms-full.txt  (nav header leaked in)');
+    const { source, text } = await readLlmsFull();
+    if (text.includes(ROOT_PARENT)) {
+        fail('llms-full.txt', `nav header leaked into ${source} (found the root breadcrumb)`);
+        console.log('❌ llms-full.txt  (nav header leaked in)');
     } else {
-        console.log('✅ /llms-full.txt  (no nav header leaked in)');
+        console.log(`✅ llms-full.txt  (no nav header leaked in; read ${source})`);
     }
 } catch (err) {
-    fail('/llms-full.txt', err.message);
-    console.log('❌ /llms-full.txt  (fetch failed)');
+    fail('llms-full.txt', err.message);
+    console.log('❌ llms-full.txt  (read failed)');
 }
 
 if (failures.length > 0) {