Charge AI training crawlers a day pass (x402 crawl gateway 0.11)

package.json

@@ -60,6 +60,7 @@
     "@profullstack/footer": "latest",
     "@profullstack/hqtui": "^0.10.3",
     "@profullstack/player": "^0.14.0",
+    "@profullstack/x402-gateway": "^0.11.0",
     "jsonwebtoken": "^9.0.2",
     "pg": "^8.23.1"
   },

src/server/crawl.js

@@ -0,0 +1,64 @@
+// The crawl door on hdtilt.com: AI training crawlers, headless browsers and SEO
+// scrapers buy a day pass ($1 a day over x402, or any coin / PayPal through
+// CoinPay's hosted checkout) on /crawl, and get a 402 everywhere else. People,
+// verified search crawlers and AI chat/search fetchers are never charged, nor is
+// a signed-in viewer. Built only with a scoped key and a payee, so a self-hosted
+// or desktop server never sells anything.
+
+import { createGateway } from '@profullstack/x402-gateway';
+
+/** @param {NodeJS.ProcessEnv} env */
+export function createCrawlGate(env = process.env) {
+  const apiKey = env.COINPAY_X402_KEY || '';
+  const payTo = env.CRAWL_PAY_TO || '';
+  if (!apiKey || !payTo) return null;
+  return createGateway({
+    siteUrl: (env.SITE_URL || 'https://hdtilt.com').replace(/\/+$/, ''),
+    siteName: 'hdtilt',
+    coinpay: { apiKey },
+    payTo,
+    contact: 'mailto:anthony@profullstack.com',
+    openPaths: ['/api/health', '/llms.txt', '/install.sh', '/.well-known/'],
+    exempt: (request) => /(?:^|;\s*)hdtilt_media=/.test(request.headers.get('cookie') ?? ''),
+  });
+}
+
+/** A fetch Request built from a Node one; the body only for the gateway's own POSTs. */
+async function requestFor(req) {
+  const headers = new Headers();
+  for (const [name, value] of Object.entries(req.headers)) {
+    if (typeof value === 'string') headers.set(name, value);
+    else if (Array.isArray(value)) headers.set(name, value.join(', '));
+  }
+  if (!headers.has('x-forwarded-for') && req.socket?.remoteAddress)
+    headers.set('x-forwarded-for', req.socket.remoteAddress);
+  const method = req.method || 'GET';
+  const url = `http://localhost${req.url || '/'}`;
+  let body;
+  if (method === 'POST' && /^\/crawl(?:\/|$)/.test(new URL(url).pathname)) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+      size += chunk.length;
+      if (size > 65536) break;
+      chunks.push(chunk);
+    }
+    body = Buffer.concat(chunks);
+  }
+  return new Request(url, { method, headers, body });
+}
+
+/** Answer the request from the gateway if it has something to say. True when it did. */
+export async function crawlAnswer(gate, req, res) {
+  const answer = await gate.handle(await requestFor(req));
+  if (!answer) return false;
+  const headers = {};
+  answer.headers.forEach((value, name) => {
+    headers[name] = value;
+  });
+  const cookies = answer.headers.getSetCookie?.() ?? [];
+  if (cookies.length) headers['set-cookie'] = cookies;
+  res.writeHead(answer.status, headers);
+  res.end(req.method === 'HEAD' ? undefined : Buffer.from(await answer.arrayBuffer()));
+  return true;
+}

src/server/server.js

@@ -19,6 +19,7 @@ import { b64urlDecode, proxyPath } from '../core/util.js';
 import { browseVod } from '../core/vod.js';
 import { VERSION } from '../version.js';
 import { createAccounts } from './accounts.js';
+import { crawlAnswer, createCrawlGate } from './crawl.js';
 import { withFooter } from './footer.js';
 import { handleRpc } from './mcp.js';
 import { createRemuxer, hasFfmpeg } from './remux.js';
@@ -55,6 +56,8 @@ export function createApp(opts = {}) {
   const allowPrivate = opts.allowPrivate ?? (process.env.HDTILT_ALLOW_PRIVATE === '1' || !isPublic);
   const maxStreams = opts.maxStreamsPerIp ?? (isPublic ? 3 : Number.POSITIVE_INFINITY);
   const streamsByIp = new Map();
+  // Training crawlers pay for a day pass on hdtilt.com; never on a local server.
+  const crawlGate = opts.crawlGate !== undefined ? opts.crawlGate : isPublic ? createCrawlGate() : null;
 
   /** fetch, routed through the SSRF guard. */
   const guardedFetch = async (url, init = {}) => {
@@ -593,6 +596,7 @@ export function createApp(opts = {}) {
     const url = new URL(req.url, 'http://x');
     const path = url.pathname;
     try {
+      if (crawlGate && !isInternal(req) && (await crawlAnswer(crawlGate, req, res))) return;
       if (req.method === 'OPTIONS') {
         return send(res, 204, '', {
           'access-control-allow-methods': 'GET, POST, PUT, OPTIONS',

test/crawl.test.js

@@ -0,0 +1,44 @@
+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
+import { createCrawlGate } from '../src/server/crawl.js';
+import { startServer } from '../src/server/server.js';
+
+const GPT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)';
+const CHROME = {
+  'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+  accept: 'text/html',
+  'sec-fetch-mode': 'navigate',
+  'sec-fetch-site': 'none',
+  'sec-fetch-dest': 'document',
+};
+let s;
+const at = (p) => `http://127.0.0.1:${s.address().port}${p}`;
+
+beforeAll(async () => {
+  const gate = createCrawlGate({
+    COINPAY_X402_KEY: 'cp_live_test',
+    CRAWL_PAY_TO: '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5',
+  });
+  s = await startServer({ port: 0, public: true, crawlGate: gate });
+});
+afterAll(() => s.close());
+
+describe('crawl gateway', () => {
+  it('is off without a key', () => {
+    expect(createCrawlGate({})).toBeNull();
+  });
+  it('charges GPTBot with a PAYMENT-REQUIRED offer', async () => {
+    const r = await fetch(at('/'), { headers: { 'user-agent': GPT } });
+    expect(r.status).toBe(402);
+    expect(r.headers.get('payment-required')).toBeTruthy();
+  });
+  it('sells a day pass on /crawl', async () => {
+    const r = await fetch(at('/crawl'), { headers: { accept: 'application/json' } });
+    expect(r.status).toBe(402);
+    const body = await r.json();
+    expect(body.accepts[0].amount).toBe('1000000');
+  });
+  it('lets a browser and the health check through', async () => {
+    expect((await fetch(at('/api/health'), { headers: { 'user-agent': GPT } })).status).toBe(200);
+    expect((await fetch(at('/'), { headers: CHROME })).status).not.toBe(402);
+  });
+});