diff --git a/bun.lock b/bun.lock index 1f298444..7534efb9 100644 --- a/bun.lock +++ b/bun.lock @@ -8,7 +8,8 @@ "@profullstack/footer": "latest", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", - "@profullstack/x402-gateway": "^0.7.0", + "@profullstack/throttle": "^0.3.2", + "@profullstack/x402-gateway": "^0.10.1", "@sentry/nextjs": "^10.56.0", "@supabase/ssr": "^0.9.0", "@supabase/supabase-js": "^2.107.0", @@ -291,7 +292,9 @@ "@profullstack/stack": ["@profullstack/stack@0.1.3", "", { "dependencies": { "@profullstack/emailer": "1.0.1", "@profullstack/referrals": "0.1.0" }, "peerDependencies": { "@supabase/ssr": ">=0.5.0", "next": ">=13.0.0", "react": ">=18.0.0" }, "optionalPeers": ["@supabase/ssr", "next", "react"] }, "sha512-NOseYE5cWMwH75/luUE46kJLyAd1xD+DSgGgwhxvgNMoJYfZ9W9vKmqdcGA/4Lgahzf7hWfjNvmLtVIzAqwhJw=="], - "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.7.0", "", { "dependencies": { "@profullstack/keys": "0.1.0" } }, "sha512-xsZRWHdOD9Y9EuRS4PsRmykVS2geOu/luY0jBlliOF3SEgxiLhgmx7ks+2jOUtGlq+cIfSonnk5eEObRGCOJ9w=="], + "@profullstack/throttle": ["@profullstack/throttle@0.3.2", "", { "dependencies": { "@profullstack/x402-gateway": "^0.6.0 || ^0.7.0 || ^0.8.0 || ^0.9.0 || ^0.10.0" } }, "sha512-53bE30J/vjuuahKiBveB3oKL+L03ufNGwmmSk9yfQwInfmC3xr9I7DvDkPti/wwAuk6V6bcfUqeazgNGLfe8rQ=="], + + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.10.1", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-YF/P3GRbb/8dAHIhDT2R7yHSAK+olrT4/zAWLzf5aF9Na8o1/rJCdP+k6UnGepyQ9RQ2qL9p5diRVwb18Zt4VA=="], "@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.0.0-rc.12", "", { "os": "android", "cpu": "arm64" }, "sha512-pv1y2Fv0JybcykuiiD3qBOBdz6RteYojRFY1d+b95WVuzx211CRh+ytI/+9iVyWQ6koTh5dawe4S/yRfOFjgaA=="], diff --git a/middleware.ts b/middleware.ts index d6c45b4d..4b948e4a 100644 --- a/middleware.ts +++ b/middleware.ts @@ -1,5 +1,6 @@ import { NextResponse, type NextRequest } from "next/server"; import { gate } from "@/lib/crawl-gateway"; +import { meter } from "@/lib/throttle"; import { trackReferralCode } from "@profullstack/stack/referrals"; export async function middleware(request: NextRequest) { @@ -8,6 +9,11 @@ export async function middleware(request: NextRequest) { // middleware.ts is the one Next runs here; src/proxy.ts is not wired. const answer = await gate(request); if (answer) return answer; + + // Then the site-wide allowance on page routes: 100 requests a minute per + // caller, and going over is answered 402 with the same offer (lib/throttle). + const overLimit = await meter(request); + if (overLimit) return overLimit; return trackReferralCode(request, NextResponse.next() as any); } diff --git a/package.json b/package.json index f6ccbc89..998c4fec 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "@profullstack/footer": "latest", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", - "@profullstack/x402-gateway": "^0.7.0", + "@profullstack/throttle": "^0.3.2", + "@profullstack/x402-gateway": "^0.10.1", "@sentry/nextjs": "^10.56.0", "@supabase/ssr": "^0.9.0", "@supabase/supabase-js": "^2.107.0", diff --git a/src/lib/throttle.ts b/src/lib/throttle.ts new file mode 100644 index 00000000..667f3cf8 --- /dev/null +++ b/src/lib/throttle.ts @@ -0,0 +1,35 @@ +/** + * The site-wide allowance: a hundred requests a minute, per caller, on every + * page route. Going over is answered 402 with the crawl gateway's offer, not + * 429. Wired 2026-10-19 (bot hits decisions: "yes" to a gateway on every site). + * + * WHY. The gate charges crawlers that say who they are. A scraper wearing a + * browser string was never counted here at all; that is the shape that walked + * 19,000 URLs a day off coinpayportal on 2026-09-08. + * + * What is never metered: /api/ (webhooks, daemons, CLIs and polling clients + * have their own limits and must never be answered with a price), a Next.js + * link prefetch (a page already loaded asking ahead for the links it shows; a + * front page full of links would otherwise spend a reader's allowance for + * them), and whatever the gateway vouches for (verified search crawlers, + * retrieval crawlers, AI chat fetchers, paid passes). + * + * Imports nothing Node-only: the middleware may run at the edge. + */ + +import { createThrottle } from "@profullstack/throttle"; +import { gateway } from "@/lib/crawl-gateway"; + +const isPrefetch = (request: Request) => + request.headers.has("next-router-prefetch") || + /prefetch/i.test(request.headers.get("sec-purpose") ?? request.headers.get("purpose") ?? ""); + +export const throttle = createThrottle({ + gateway, + exempt: isPrefetch, + credential: { limit: 600, ceiling: 1200 }, + rules: [{ path: "/api/", open: true }], +}); + +/** Resolves to a Response for a caller over the allowance, or undefined. */ +export const meter = (request: Request) => throttle.handle(request); diff --git a/src/proxy.ts b/src/proxy.ts index ce309ae8..6c65396a 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -1,4 +1,5 @@ import { gate } from "@/lib/crawl-gateway"; +import { meter } from "@/lib/throttle"; import type { NextRequest } from "next/server"; import { NextResponse } from "next/server"; import { updateSession } from "@/lib/supabase/middleware"; @@ -15,6 +16,11 @@ export async function proxy(request: NextRequest) { const answer = await gate(request); if (answer) return answer; + // Then the site-wide allowance on page routes: 100 requests a minute per + // caller, and going over is answered 402 with the same offer (lib/throttle). + const overLimit = await meter(request); + if (overLimit) return overLimit; + const hostname = request.headers.get("host") || ""; // Redirect www to non-www