Fix ad reporting and throttle crawlers through x402 gateway

.github/workflows/ci.yml

@@ -31,3 +31,8 @@ jobs:
 
       - name: Tests
         run: npm test
+
+      - name: Ad reporting and crawler limits (PostgreSQL + Redis)
+        run: |
+          node scripts/test-ad-token-earnings.mjs
+          node --import tsx scripts/test-crawl-limits.mjs

app/(app)/dashboard/ads/earnings/page.tsx

@@ -21,6 +21,8 @@ const EMPTY: EarningsModel = {
   // Signed out: nothing was attempted, so nothing failed.
   statsUnavailable: false,
   totals: {
+    advBilledClicks: 0, advFreeClicks: 0, pubBilledClicks: 0, pubFreeClicks: 0,
+    advPaidImpressions: 0, advFreeImpressions: 0, pubPaidImpressions: 0, pubFreeImpressions: 0,
     spentCents: 0,
     earnedCents: 0,
     netCents: 0,

app/a/[id]/route.ts

@@ -17,6 +17,8 @@ import { adClickIp } from "@/lib/ads/client-ip";
 import { parseDevice } from "@/lib/tracker/device";
 import { isShortCode } from "@/lib/ads/shortcode";
 import { env } from "@/lib/env";
+import { gate } from "@/lib/crawl-gateway";
+import { crawlerFamily } from "@/lib/crawl-policy";
 
 export const runtime = "nodejs";
 export const dynamic = "force-dynamic";
@@ -67,6 +69,8 @@ async function findImpression(
 }
 
 export async function GET(request: NextRequest, ctx: { params: Promise<{ id: string }> }) {
+  const denied = await gate(request);
+  if (denied) return denied;
   const fallback = env.siteUrl || "https://crawlproof.com";
   try {
     const { id } = await ctx.params;
@@ -82,12 +86,13 @@ export async function GET(request: NextRequest, ctx: { params: Promise<{ id: str
     if (!imp) return NextResponse.redirect(fallback, { status: 302 });
 
     const ip = adClickIp(request.headers);
-    const geo = await lookupGeo(ip).catch(() => null);
+    const crawler = crawlerFamily(request.headers.get("user-agent"));
+    const geo = crawler ? null : await lookupGeo(ip).catch(() => null);
     // Deliberately the STRICT classification here, unlike /api/ads/motd: a
     // terminal ad is served to curl, but it's clicked from a browser when the
     // reader follows the link. Anyone can curl this URL in a loop, so scripted
     // hits stay unbilled (recorded with valid=false) rather than paying out.
-    const device = parseDevice(request.headers.get("user-agent")).deviceType;
+    const device = crawler ? "bot" : parseDevice(request.headers.get("user-agent")).deviceType;
 
     const dest = await resolveClick({
       impressionId: imp.id,
@@ -103,6 +108,7 @@ export async function GET(request: NextRequest, ctx: { params: Promise<{ id: str
     });
 
     if (!dest) return NextResponse.redirect(fallback, { status: 302 });
+    if (crawler) return NextResponse.redirect(dest, { status: 302, headers: { "cache-control": "no-store", "x-robots-tag": "noindex, nofollow" } });
 
     // Terminal traffic is invisible in an advertiser's analytics without a tag
     // — there's no referrer from a shell. resolveClick already appended

app/api/admin/crawl-activity/route.ts

@@ -0,0 +1,30 @@
+import { NextRequest, NextResponse } from "next/server";
+import { authenticateBearer } from "@/lib/sp/apiAuth";
+import { createClient } from "@/lib/supabase/server";
+import { serviceClient } from "@/lib/supabase/service";
+import { readCrawlerActivity } from "@/lib/crawl-limits";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export async function GET(request: NextRequest) {
+  let userId: string | undefined;
+  if (request.headers.has("authorization")) {
+    const auth = await authenticateBearer(request);
+    if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status });
+    userId = auth.userId;
+  } else {
+    const sb = await createClient();
+    userId = (await sb.auth.getUser()).data.user?.id;
+  }
+  if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+  const { data: profile, error } = await serviceClient().from("profiles").select("is_admin").eq("id", userId).maybeSingle();
+  if (error || !profile?.is_admin) return NextResponse.json({ error: "Forbidden" }, { status: 403 });
+  const requested = Number(request.nextUrl.searchParams.get("days") ?? 7);
+  const days = Number.isInteger(requested) ? Math.min(31, Math.max(1, requested)) : 7;
+  try {
+    return NextResponse.json({ scope: "network", rangeDays: days, daily: await readCrawlerActivity(days) }, { headers: { "cache-control": "private, no-store" } });
+  } catch {
+    return NextResponse.json({ error: "Crawler activity is temporarily unavailable" }, { status: 503 });
+  }
+}

app/api/ads/click/route.ts

@@ -8,11 +8,15 @@ import { lookupGeo } from "@/lib/tracker/geo";
 import { adClickIp } from "@/lib/ads/client-ip";
 import { parseDevice } from "@/lib/tracker/device";
 import { env } from "@/lib/env";
+import { gate } from "@/lib/crawl-gateway";
+import { crawlerFamily } from "@/lib/crawl-policy";
 
 export const runtime = "nodejs";
 export const dynamic = "force-dynamic";
 
 export async function GET(request: NextRequest) {
+  const denied = await gate(request);
+  if (denied) return denied;
   const fallback = env.siteUrl || "https://crawlproof.com";
   try {
     const url = new URL(request.url);
@@ -23,8 +27,9 @@ export async function GET(request: NextRequest) {
     const visitorId = url.searchParams.get("v");
 
     const ip = adClickIp(request.headers);
-    const geo = await lookupGeo(ip).catch(() => null);
-    const device = parseDevice(request.headers.get("user-agent")).deviceType;
+    const crawler = crawlerFamily(request.headers.get("user-agent"));
+    const geo = crawler ? null : await lookupGeo(ip).catch(() => null);
+    const device = crawler ? "bot" : parseDevice(request.headers.get("user-agent")).deviceType;
 
     const dest = await resolveClick({
       impressionId,

app/api/ads/v1/earnings/route.ts

@@ -1,134 +1,32 @@
-// /api/ads/v1/earnings — the account's ad money and delivery, for a token caller.
-//
-//   GET ?days=7|30|90|365
-//
-// The same model /dashboard/ads/earnings renders, for something holding an API
-// token. It exists because the alternative for a client that wants fleet totals
-// is one /campaigns/[id] request per campaign, and the account is past 170 of
-// them — see `crawlproof dashboard`, which polls this on a timer.
-//
-// Same auth as the rest of /api/ads/v1/*: `Authorization: Bearer crp_…`.
-
 import { NextResponse, type NextRequest } from "next/server";
-
 import { serviceClient } from "@/lib/supabase/service";
 import { authenticateBearer } from "@/lib/sp/apiAuth";
 import { loadEarnings } from "@/lib/ads/earnings-data";
+import { loadTokenDelivery } from "@/lib/ads/token-earnings";
 
 export const runtime = "nodejs";
 export const dynamic = "force-dynamic";
 
-const ALLOWED_DAYS = [7, 30, 90, 365];
-
-/** An unknown window falls back to 30 rather than reaching the query planner. */
 export function parseDays(raw: string | null): number {
   const n = Number(raw);
-  return ALLOWED_DAYS.includes(n) ? n : 30;
+  return [7, 30, 90, 365].includes(n) ? n : 30;
 }
 
 export async function GET(req: NextRequest) {
   const auth = await authenticateBearer(req);
   if (!auth.ok) return NextResponse.json({ error: auth.error }, { status: auth.status });
-
   const days = parseDays(req.nextUrl.searchParams.get("days"));
-  const sb = serviceClient();
-  // The service client has no RLS. loadEarnings filters every table by
-  // owner_id itself, which is what makes passing it here safe.
-  const model = await loadEarnings(sb, auth.userId, days);
-
-  // The windowed delivery figures come from RPCs that are `security definer`
-  // and filter on `auth.uid()`. A service client has no auth.uid(), so they
-  // return nothing and every impression count arrives as a confident zero.
-  // The stats views are `security_invoker` and granted to service_role, so
-  // they can be read directly. They are lifetime rather than windowed, which
-  // is why this only replaces figures that came back empty, and why the answer
-  // says which it gave you.
-  const delivery = await lifetimeDelivery(sb, model);
-  return NextResponse.json({ ...model, ...delivery });
-}
-
-type StatsRow = {
-  impressions: number | null;
-  free_impressions: number | null;
-  clicks: number | null;
-  free_clicks: number | null;
-};
-
-/**
- * What the stats views actually mean, which is not what the column names
- * suggest and is worth writing down once:
- *
- *   impressions       paid tier, non-duplicate
- *   free_impressions  free tier, non-duplicate  -> delivery is the SUM of both
- *   clicks            **valid** clicks, either tier -> delivery is this alone
- *   free_clicks       free tier and **not valid** -> fraud/duplicate, NOT delivery
- *
- * So impressions add up and clicks do not. Adding `free_clicks` into clicks
- * would fold invalid clicks into the CTR, which is the one number a click
- * fraud problem would show up in.
- */
-const sum = (rows: StatsRow[], key: keyof StatsRow) =>
-  rows.reduce((total, row) => total + (Number(row[key]) || 0), 0);
-
-/**
- * Read a stats view for a list of ids, in chunks.
- *
- * PostgREST puts `in.(…)` in the query string, and this account is past 180
- * campaigns, so one call is a 7KB URL that comes back empty rather than
- * erroring. That empty answer is exactly what a network with no delivery looks
- * like, which is how it went unnoticed.
- */
-async function readStats(
-  sb: ReturnType<typeof serviceClient>,
-  view: "ad_campaign_stats" | "ad_slot_stats",
-  key: "campaign_id" | "slot_id",
-  ids: string[],
-): Promise<{ rows: StatsRow[]; failed: boolean }> {
-  const rows: StatsRow[] = [];
-  let failed = false;
-  const CHUNK = 50;
-  for (let i = 0; i < ids.length; i += CHUNK) {
-    const { data, error } = await sb
-      .from(view)
-      .select("impressions, free_impressions, clicks, free_clicks")
-      .in(key, ids.slice(i, i + CHUNK));
-    if (error) failed = true;
-    else rows.push(...((data ?? []) as StatsRow[]));
+  try {
+    const sb = serviceClient();
+    const model = await loadEarnings(sb, auth.userId, days, loadTokenDelivery(sb, auth.userId, days));
+    if (model.statsUnavailable) throw new Error("incomplete_reporting");
+    return NextResponse.json({ ...model, deliveryWindow: "range", clickSemantics: "accepted_billed_plus_free" }, {
+      headers: { "cache-control": "private, no-store" },
+    });
+  } catch {
+    console.error("[ads] Earnings response unavailable");
+    return NextResponse.json({ error: "Ad reporting is temporarily unavailable. Please retry.", statsUnavailable: true }, {
+      status: 503, headers: { "retry-after": "5", "cache-control": "no-store" },
+    });
   }
-  return { rows, failed };
-}
-
-async function lifetimeDelivery(
-  sb: ReturnType<typeof serviceClient>,
-  model: Awaited<ReturnType<typeof loadEarnings>>,
-) {
-  const t = model.totals;
-  const empty = !t.advImpressions && !t.advClicks && !t.pubImpressions && !t.pubClicks;
-  if (!empty) return { deliveryWindow: "range" as const };
-
-  const campaignIds = model.campaigns.map((c) => c.id);
-  const slotIds = model.slots.map((s) => s.id);
-  if (!campaignIds.length && !slotIds.length) return { deliveryWindow: "range" as const };
-
-  const [c, s] = await Promise.all([
-    readStats(sb, "ad_campaign_stats", "campaign_id", campaignIds),
-    readStats(sb, "ad_slot_stats", "slot_id", slotIds),
-  ]);
-
-  return {
-    deliveryWindow: "lifetime" as const,
-    statsUnavailable: model.statsUnavailable || c.failed || s.failed,
-    totals: {
-      ...t,
-      advImpressions: sum(c.rows, "impressions") + sum(c.rows, "free_impressions"),
-      advClicks: sum(c.rows, "clicks"),
-      advFreeImpressions: sum(c.rows, "free_impressions"),
-      advPaidImpressions: sum(c.rows, "impressions"),
-      pubImpressions: sum(s.rows, "impressions") + sum(s.rows, "free_impressions"),
-      pubClicks: sum(s.rows, "clicks"),
-      pubFreeImpressions: sum(s.rows, "free_impressions"),
-      pubPaidImpressions: sum(s.rows, "impressions"),
-      invalidClicks: sum(c.rows, "free_clicks"),
-    },
-  };
 }

app/robots.txt/route.ts

@@ -5,5 +5,5 @@ import { gateway } from "@/lib/crawl-gateway";
 // crawlers are refused everywhere but /crawl (where they can buy a pass),
 // retrieval crawlers are named as welcome, everyone else gets the rules below.
 export const GET = robotsRoute(gateway, {
-  disallow: ["/api/"],
+  disallow: ["/api/", "/a/"],
 });