diff --git a/bun.lock b/bun.lock index 7ad76ad..6fb0acc 100644 --- a/bun.lock +++ b/bun.lock @@ -10,6 +10,7 @@ "@profullstack/footer": "latest", "@profullstack/hqtui": "^0.10.3", "@profullstack/player": "^0.14.0", + "@profullstack/x402-gateway": "^0.11.0", "jsonwebtoken": "^9.0.2", "pg": "^8.23.1", }, @@ -85,8 +86,12 @@ "@profullstack/hqtui": ["@profullstack/hqtui@0.10.3", "", { "bin": { "hqtui": "bin/hqtui.mjs" } }, "sha512-Z2jaDqmiaGyrMy4JXOoPGCyEkei8l3tHoCbmT3/1gAWDl72ynPvMuHcUj+U97KgsF/CFIpZzzBYnYqcOq5e46g=="], + "@profullstack/keys": ["@profullstack/keys@0.1.0", "", {}, "sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w=="], + "@profullstack/player": ["@profullstack/player@0.14.0", "", { "dependencies": { "hls.js": "^1.7.1", "mpegts.js": "^1.8.2" }, "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"] }, "sha512-HBDNkb17jGhZrkpksCp1iB07yWjfKIeDQq8LarhA04bKBZdX+vkRRTN1gsxXiAM3T4t0Zxj+99dpTA0uPtKofw=="], + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.11.0", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-Lfrz+SRTwQZfcoCaVtaaMNPJx74Lp/5oyS7vJJAEuJmH1TjBzhE7dWquISkj1p42oc+VW5I9xH9p7YhEKqmr4Q=="], + "@resvg/resvg-js": ["@resvg/resvg-js@2.6.2", "", { "optionalDependencies": { "@resvg/resvg-js-android-arm-eabi": "2.6.2", "@resvg/resvg-js-android-arm64": "2.6.2", "@resvg/resvg-js-darwin-arm64": "2.6.2", "@resvg/resvg-js-darwin-x64": "2.6.2", "@resvg/resvg-js-linux-arm-gnueabihf": "2.6.2", "@resvg/resvg-js-linux-arm64-gnu": "2.6.2", "@resvg/resvg-js-linux-arm64-musl": "2.6.2", "@resvg/resvg-js-linux-x64-gnu": "2.6.2", "@resvg/resvg-js-linux-x64-musl": "2.6.2", "@resvg/resvg-js-win32-arm64-msvc": "2.6.2", "@resvg/resvg-js-win32-ia32-msvc": "2.6.2", "@resvg/resvg-js-win32-x64-msvc": "2.6.2" } }, "sha512-xBaJish5OeGmniDj9cW5PRa/PtmuVU3ziqrbr5xJj901ZDN4TosrVaNZpEiLZAxdfnhAe7uQ7QFWfjPe9d9K2Q=="], "@resvg/resvg-js-android-arm-eabi": ["@resvg/resvg-js-android-arm-eabi@2.6.2", "", { "os": "android", "cpu": "arm" }, "sha512-FrJibrAk6v29eabIPgcTUMPXiEz8ssrAk7TXxsiZzww9UTQ1Z5KAbFJs+Z0Ez+VZTYgnE5IQJqBcoSiMebtPHA=="], diff --git a/package.json b/package.json index 7ab6170..625fec2 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "@profullstack/footer": "latest", "@profullstack/hqtui": "^0.10.3", "@profullstack/player": "^0.14.0", + "@profullstack/x402-gateway": "^0.11.0", "jsonwebtoken": "^9.0.2", "pg": "^8.23.1" }, diff --git a/src/server/crawl.js b/src/server/crawl.js new file mode 100644 index 0000000..23ede45 --- /dev/null +++ b/src/server/crawl.js @@ -0,0 +1,64 @@ +// The crawl door on hdtilt.com: AI training crawlers, headless browsers and SEO +// scrapers buy a day pass ($1 a day over x402, or any coin / PayPal through +// CoinPay's hosted checkout) on /crawl, and get a 402 everywhere else. People, +// verified search crawlers and AI chat/search fetchers are never charged, nor is +// a signed-in viewer. Built only with a scoped key and a payee, so a self-hosted +// or desktop server never sells anything. + +import { createGateway } from '@profullstack/x402-gateway'; + +/** @param {NodeJS.ProcessEnv} env */ +export function createCrawlGate(env = process.env) { + const apiKey = env.COINPAY_X402_KEY || ''; + const payTo = env.CRAWL_PAY_TO || ''; + if (!apiKey || !payTo) return null; + return createGateway({ + siteUrl: (env.SITE_URL || 'https://hdtilt.com').replace(/\/+$/, ''), + siteName: 'hdtilt', + coinpay: { apiKey }, + payTo, + contact: 'mailto:anthony@profullstack.com', + openPaths: ['/api/health', '/llms.txt', '/install.sh', '/.well-known/'], + exempt: (request) => /(?:^|;\s*)hdtilt_media=/.test(request.headers.get('cookie') ?? ''), + }); +} + +/** A fetch Request built from a Node one; the body only for the gateway's own POSTs. */ +async function requestFor(req) { + const headers = new Headers(); + for (const [name, value] of Object.entries(req.headers)) { + if (typeof value === 'string') headers.set(name, value); + else if (Array.isArray(value)) headers.set(name, value.join(', ')); + } + if (!headers.has('x-forwarded-for') && req.socket?.remoteAddress) + headers.set('x-forwarded-for', req.socket.remoteAddress); + const method = req.method || 'GET'; + const url = `http://localhost${req.url || '/'}`; + let body; + if (method === 'POST' && /^\/crawl(?:\/|$)/.test(new URL(url).pathname)) { + const chunks = []; + let size = 0; + for await (const chunk of req) { + size += chunk.length; + if (size > 65536) break; + chunks.push(chunk); + } + body = Buffer.concat(chunks); + } + return new Request(url, { method, headers, body }); +} + +/** Answer the request from the gateway if it has something to say. True when it did. */ +export async function crawlAnswer(gate, req, res) { + const answer = await gate.handle(await requestFor(req)); + if (!answer) return false; + const headers = {}; + answer.headers.forEach((value, name) => { + headers[name] = value; + }); + const cookies = answer.headers.getSetCookie?.() ?? []; + if (cookies.length) headers['set-cookie'] = cookies; + res.writeHead(answer.status, headers); + res.end(req.method === 'HEAD' ? undefined : Buffer.from(await answer.arrayBuffer())); + return true; +} diff --git a/src/server/server.js b/src/server/server.js index d29c595..aab7ea0 100644 --- a/src/server/server.js +++ b/src/server/server.js @@ -19,6 +19,7 @@ import { b64urlDecode, proxyPath } from '../core/util.js'; import { browseVod } from '../core/vod.js'; import { VERSION } from '../version.js'; import { createAccounts } from './accounts.js'; +import { crawlAnswer, createCrawlGate } from './crawl.js'; import { withFooter } from './footer.js'; import { handleRpc } from './mcp.js'; import { createRemuxer, hasFfmpeg } from './remux.js'; @@ -55,6 +56,8 @@ export function createApp(opts = {}) { const allowPrivate = opts.allowPrivate ?? (process.env.HDTILT_ALLOW_PRIVATE === '1' || !isPublic); const maxStreams = opts.maxStreamsPerIp ?? (isPublic ? 3 : Number.POSITIVE_INFINITY); const streamsByIp = new Map(); + // Training crawlers pay for a day pass on hdtilt.com; never on a local server. + const crawlGate = opts.crawlGate !== undefined ? opts.crawlGate : isPublic ? createCrawlGate() : null; /** fetch, routed through the SSRF guard. */ const guardedFetch = async (url, init = {}) => { @@ -593,6 +596,7 @@ export function createApp(opts = {}) { const url = new URL(req.url, 'http://x'); const path = url.pathname; try { + if (crawlGate && !isInternal(req) && (await crawlAnswer(crawlGate, req, res))) return; if (req.method === 'OPTIONS') { return send(res, 204, '', { 'access-control-allow-methods': 'GET, POST, PUT, OPTIONS', diff --git a/test/crawl.test.js b/test/crawl.test.js new file mode 100644 index 0000000..bcd8838 --- /dev/null +++ b/test/crawl.test.js @@ -0,0 +1,44 @@ +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { createCrawlGate } from '../src/server/crawl.js'; +import { startServer } from '../src/server/server.js'; + +const GPT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)'; +const CHROME = { + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', + accept: 'text/html', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-dest': 'document', +}; +let s; +const at = (p) => `http://127.0.0.1:${s.address().port}${p}`; + +beforeAll(async () => { + const gate = createCrawlGate({ + COINPAY_X402_KEY: 'cp_live_test', + CRAWL_PAY_TO: '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5', + }); + s = await startServer({ port: 0, public: true, crawlGate: gate }); +}); +afterAll(() => s.close()); + +describe('crawl gateway', () => { + it('is off without a key', () => { + expect(createCrawlGate({})).toBeNull(); + }); + it('charges GPTBot with a PAYMENT-REQUIRED offer', async () => { + const r = await fetch(at('/'), { headers: { 'user-agent': GPT } }); + expect(r.status).toBe(402); + expect(r.headers.get('payment-required')).toBeTruthy(); + }); + it('sells a day pass on /crawl', async () => { + const r = await fetch(at('/crawl'), { headers: { accept: 'application/json' } }); + expect(r.status).toBe(402); + const body = await r.json(); + expect(body.accepts[0].amount).toBe('1000000'); + }); + it('lets a browser and the health check through', async () => { + expect((await fetch(at('/api/health'), { headers: { 'user-agent': GPT } })).status).toBe(200); + expect((await fetch(at('/'), { headers: CHROME })).status).not.toBe(402); + }); +});