From 813e7addac673a65c6da3f042ebae356e51e4630 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 21 Oct 2026 14:35:29 +0000 Subject: [PATCH] feat: x402 crawl gateway for training crawlers Training crawlers get a 402 with PAYMENT-REQUIRED and /crawl sells a $1 day pass (x402 USDC or CoinPay checkout). Browsers, verified search crawlers, AI chat fetchers and MCP clients pass through; /healthz and /api/ stay open. The proxy matcher now covers every page so the gateway sees them. Co-Authored-By: Claude Opus 5.5 --- bun.lock | 5 +++++ package.json | 1 + src/lib/crawl-gateway.test.ts | 39 +++++++++++++++++++++++++++++++++++ src/lib/crawl-gateway.ts | 25 ++++++++++++++++++++++ src/proxy.ts | 11 ++++++++-- 5 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 src/lib/crawl-gateway.test.ts create mode 100644 src/lib/crawl-gateway.ts diff --git a/bun.lock b/bun.lock index 6a1e923..873ae7c 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,7 @@ "name": "datamart-web", "dependencies": { "@profullstack/footer": "latest", + "@profullstack/x402-gateway": "^0.11.0", "hono": "4.13.12", "next": "16.3.8", "react": "19.2.3", @@ -97,6 +98,10 @@ "@profullstack/footer": ["@profullstack/footer@0.2.4", "", { "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"], "bin": { "profullstack-footer": "bin/profullstack-footer.js" } }, "sha512-2ErIzHJCDA2baP8aK6f7rCHohW4EX+wnp7D1mrVMQTUxRcyZwKzmho96hhEFJndXRBGgxT3H/78BHOeBfLxTzg=="], + "@profullstack/keys": ["@profullstack/keys@0.1.0", "", {}, "sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w=="], + + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.11.0", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-Lfrz+SRTwQZfcoCaVtaaMNPJx74Lp/5oyS7vJJAEuJmH1TjBzhE7dWquISkj1p42oc+VW5I9xH9p7YhEKqmr4Q=="], + "@swc/helpers": ["@swc/helpers@0.5.23", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw=="], "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], diff --git a/package.json b/package.json index 374b719..10311f2 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@profullstack/footer": "latest", + "@profullstack/x402-gateway": "^0.11.0", "hono": "4.13.12", "next": "16.3.8", "react": "19.2.3", diff --git a/src/lib/crawl-gateway.test.ts b/src/lib/crawl-gateway.test.ts new file mode 100644 index 0000000..75472b3 --- /dev/null +++ b/src/lib/crawl-gateway.test.ts @@ -0,0 +1,39 @@ +import { beforeAll, expect, test } from "bun:test"; + +const GPTBOT = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)"; +const HUMAN = { + "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36", + accept: "text/html,application/xhtml+xml", + "accept-language": "en-US", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "sec-fetch-dest": "document", + "x-real-ip": "198.51.100.7", +}; + +let gateway: { handle: (r: Request) => Promise }; +beforeAll(async () => { + process.env.COINPAY_X402_KEY = "cp_live_test"; + process.env.CRAWL_PAY_TO = "0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5"; + ({ gateway } = await import("./crawl-gateway")); +}); + +test("a training crawler is answered 402 with PAYMENT-REQUIRED", async () => { + const res = await gateway.handle(new Request("https://datamart.help/gov", { headers: { "user-agent": GPTBOT, "x-real-ip": "203.0.113.9" } })); + expect(res?.status).toBe(402); + expect(res?.headers.get("payment-required")).toBeTruthy(); +}); + +test("/crawl offers a $1 day pass", async () => { + const res = await gateway.handle(new Request("https://datamart.help/crawl", { headers: { accept: "application/json" } })); + expect(res?.status).toBe(402); + expect(res?.headers.get("payment-required")).toBeTruthy(); + const body = (await res!.json()) as { accepts: { amount: string }[] }; + expect(String(body.accepts[0]?.amount)).toBe("1000000"); +}); + +test("browsers, the API and MCP endpoints pass through", async () => { + expect(await gateway.handle(new Request("https://datamart.help/", { headers: HUMAN }))).toBeNull(); + expect(await gateway.handle(new Request("https://datamart.help/api/v1/health", { headers: { "user-agent": GPTBOT } }))).toBeNull(); + expect(await gateway.handle(new Request("https://datamart.help/gov/mcp", { method: "POST", headers: { "user-agent": GPTBOT } }))).toBeNull(); +}); diff --git a/src/lib/crawl-gateway.ts b/src/lib/crawl-gateway.ts new file mode 100644 index 0000000..c429b18 --- /dev/null +++ b/src/lib/crawl-gateway.ts @@ -0,0 +1,25 @@ +import { createGateway } from "@profullstack/x402-gateway"; +import { x402Proxy } from "@profullstack/x402-gateway/next"; +import { isMcpPath } from "./passthrough"; +import { SITE } from "./site"; + +// Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, ...) by the +// day over x402, settled by CoinPay in USDC, with a CoinPay checkout for people at +// /crawl. Browsers, Googlebot, AI chat/search fetchers and MCP clients pass through. +// Read through a computed key: Next inlines a literal process.env.NAME at build time. +const env = (name: string): string | undefined => process.env[name]; + +export const gateway = createGateway({ + siteUrl: (env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || SITE).replace(/\/+$/, ""), + siteName: "Datamart", + coinpay: { apiKey: env("COINPAY_X402_KEY") }, + payTo: env("CRAWL_PAY_TO"), + contact: "mailto:anthony@profullstack.com", + // Health checks, the versioned API and the crawl manifests are never charged. + openPaths: ["/healthz", "/api/", "/api", "/robots.txt", "/sitemap.xml", "/llms.txt", "/.well-known/", "/manifest.webmanifest", "/sw.js"], + // MCP endpoints are the product for agents: never charged. + exempt: (request: Request) => isMcpPath(new URL(request.url).pathname), +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/src/proxy.ts b/src/proxy.ts index 6376961..506e771 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -1,10 +1,16 @@ import { NextResponse, type NextRequest } from "next/server"; import { shouldPassThrough } from "@/lib/passthrough"; import { upstreamBase } from "@/lib/upstream"; +import { gate } from "@/lib/crawl-gateway"; // Agents, scripts and MCP clients get the Datamart API response on the same URL a // person opens in a browser. Everything else renders the web pages. -export function proxy(req: NextRequest) { +export async function proxy(req: NextRequest) { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the pass + // page at /crawl) unless they present a paid pass. Everyone else carries on. + const answer = await gate(req); + if (answer) return answer; + const { pathname, searchParams } = req.nextUrl; const raw = searchParams.has("raw"); if (!shouldPassThrough(req.method, pathname, req.headers.get("accept"), raw)) return NextResponse.next(); @@ -14,5 +20,6 @@ export function proxy(req: NextRequest) { } export const config = { - matcher: ["/search/:path*", "/gov/:path*", "/lib/:path*", "/edu/:path*", "/contracts/:path*", "/finance/:path*"], + // Every page (the crawl gateway) minus static build output and images. + matcher: ["/((?!_next/static|_next/image|favicon.ico|.*\\.(?:svg|png|jpg|jpeg|gif|webp|avif|ico|woff2?|css|js|map)$).*)"], };