feat: x402 crawl gateway for training crawlers

package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@profullstack/footer": "latest",
+    "@profullstack/x402-gateway": "^0.11.0",
     "hono": "4.13.12",
     "next": "16.3.8",
     "react": "19.2.3",

src/lib/crawl-gateway.test.ts

@@ -0,0 +1,39 @@
+import { beforeAll, expect, test } from "bun:test";
+
+const GPTBOT = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)";
+const HUMAN = {
+  "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+  accept: "text/html,application/xhtml+xml",
+  "accept-language": "en-US",
+  "sec-fetch-mode": "navigate",
+  "sec-fetch-site": "none",
+  "sec-fetch-dest": "document",
+  "x-real-ip": "198.51.100.7",
+};
+
+let gateway: { handle: (r: Request) => Promise<Response | null> };
+beforeAll(async () => {
+  process.env.COINPAY_X402_KEY = "cp_live_test";
+  process.env.CRAWL_PAY_TO = "0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5";
+  ({ gateway } = await import("./crawl-gateway"));
+});
+
+test("a training crawler is answered 402 with PAYMENT-REQUIRED", async () => {
+  const res = await gateway.handle(new Request("https://datamart.help/gov", { headers: { "user-agent": GPTBOT, "x-real-ip": "203.0.113.9" } }));
+  expect(res?.status).toBe(402);
+  expect(res?.headers.get("payment-required")).toBeTruthy();
+});
+
+test("/crawl offers a $1 day pass", async () => {
+  const res = await gateway.handle(new Request("https://datamart.help/crawl", { headers: { accept: "application/json" } }));
+  expect(res?.status).toBe(402);
+  expect(res?.headers.get("payment-required")).toBeTruthy();
+  const body = (await res!.json()) as { accepts: { amount: string }[] };
+  expect(String(body.accepts[0]?.amount)).toBe("1000000");
+});
+
+test("browsers, the API and MCP endpoints pass through", async () => {
+  expect(await gateway.handle(new Request("https://datamart.help/", { headers: HUMAN }))).toBeNull();
+  expect(await gateway.handle(new Request("https://datamart.help/api/v1/health", { headers: { "user-agent": GPTBOT } }))).toBeNull();
+  expect(await gateway.handle(new Request("https://datamart.help/gov/mcp", { method: "POST", headers: { "user-agent": GPTBOT } }))).toBeNull();
+});

src/lib/crawl-gateway.ts

@@ -0,0 +1,25 @@
+import { createGateway } from "@profullstack/x402-gateway";
+import { x402Proxy } from "@profullstack/x402-gateway/next";
+import { isMcpPath } from "./passthrough";
+import { SITE } from "./site";
+
+// Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, ...) by the
+// day over x402, settled by CoinPay in USDC, with a CoinPay checkout for people at
+// /crawl. Browsers, Googlebot, AI chat/search fetchers and MCP clients pass through.
+// Read through a computed key: Next inlines a literal process.env.NAME at build time.
+const env = (name: string): string | undefined => process.env[name];
+
+export const gateway = createGateway({
+  siteUrl: (env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || SITE).replace(/\/+$/, ""),
+  siteName: "Datamart",
+  coinpay: { apiKey: env("COINPAY_X402_KEY") },
+  payTo: env("CRAWL_PAY_TO"),
+  contact: "mailto:anthony@profullstack.com",
+  // Health checks, the versioned API and the crawl manifests are never charged.
+  openPaths: ["/healthz", "/api/", "/api", "/robots.txt", "/sitemap.xml", "/llms.txt", "/.well-known/", "/manifest.webmanifest", "/sw.js"],
+  // MCP endpoints are the product for agents: never charged.
+  exempt: (request: Request) => isMcpPath(new URL(request.url).pathname),
+});
+
+/** Resolves to a Response for a refused crawler, or undefined to carry on. */
+export const gate = x402Proxy(gateway);

src/proxy.ts

@@ -1,10 +1,16 @@
 import { NextResponse, type NextRequest } from "next/server";
 import { shouldPassThrough } from "@/lib/passthrough";
 import { upstreamBase } from "@/lib/upstream";
+import { gate } from "@/lib/crawl-gateway";
 
 // Agents, scripts and MCP clients get the Datamart API response on the same URL a
 // person opens in a browser. Everything else renders the web pages.
-export function proxy(req: NextRequest) {
+export async function proxy(req: NextRequest) {
+  // Crawl gateway first: AI training crawlers get 402 Payment Required (or the pass
+  // page at /crawl) unless they present a paid pass. Everyone else carries on.
+  const answer = await gate(req);
+  if (answer) return answer;
+
   const { pathname, searchParams } = req.nextUrl;
   const raw = searchParams.has("raw");
   if (!shouldPassThrough(req.method, pathname, req.headers.get("accept"), raw)) return NextResponse.next();
@@ -14,5 +20,6 @@ export function proxy(req: NextRequest) {
 }
 
 export const config = {
-  matcher: ["/search/:path*", "/gov/:path*", "/lib/:path*", "/edu/:path*", "/contracts/:path*", "/finance/:path*"],
+  // Every page (the crawl gateway) minus static build output and images.
+  matcher: ["/((?!_next/static|_next/image|favicon.ico|.*\\.(?:svg|png|jpg|jpeg|gif|webp|avif|ico|woff2?|css|js|map)$).*)"],
 };