feat: persist permanent webhook delivery failures

README.md

@@ -42,12 +42,13 @@ Aggregates price data from Stellar's Classic Order Book (SDEX) and AMM Liquidity
 | POST | `/settle` | Settle an x402 payment (idempotent — retries replay the stored result) |
 | POST | `/webhooks` | Subscribe to a price-threshold webhook |
 | DELETE | `/webhooks/:id` | Delete a webhook subscription |
+| GET | `/admin/webhook-delivery-failures?limit=25` | Recent permanent webhook failures (requires `X-Admin-Token`) |
 | GET/POST | `/graphql` | GraphQL query/mutation endpoint (GraphiQL IDE at `/graphiql`, docs: [`docs/graphql.md`](docs/graphql.md)) |
 
 Every non-internal route above has a matching entry in [`openapi.yaml`](openapi.yaml);
 `tests/openapi.test.ts` boots the server and fails if a registered route is
 missing from the spec. Operator-only and non-HTTP routes (`/admin/keys*`,
-`/admin/usage*`, `/metrics`, `/ws`, `/graphiql*`) are deliberately excluded via
+`/admin/usage*`, `/admin/webhook-delivery-failures`, `/metrics`, `/ws`, `/graphiql*`) are deliberately excluded via
 the commented allow-list in [`src/openapi/coverage.ts`](src/openapi/coverage.ts).
 
 The `?network=testnet\|mainnet` query param (or `x-network` header) selects the

docs/cookbook/alert-bot.md

@@ -183,6 +183,15 @@ Delete it with `DELETE /webhooks/:id`. Use the **bot** when you want local
 control/custom logic, and **webhooks** when you want Lens to hold the
 subscription.
 
+If delivery permanently fails, Lens stores the endpoint host, alert
+payload, attempt count, final HTTP status (when available), failure category,
+and first/last attempt times. URL credentials, paths, query parameters,
+fragments, and the HMAC secret are excluded from the stored endpoint/record. Recent failures are
+available to operators at `GET /admin/webhook-delivery-failures` with an
+`X-Admin-Token` header; `limit` defaults to 25 and is capped at 100. Records are
+retained for 30 days. They are diagnostic only and are not replayed
+automatically, since a delayed threshold alert can be stale.
+
 ## Live demo
 
 - **Interactive API explorer (GraphiQL):** run Lens locally and open

prisma/schema.prisma

@@ -256,6 +256,23 @@ model Webhook {
   @@map("webhooks")
 }
 
+/// Permanent webhook failures are retained for 30-day diagnosis only; replay is
+/// not automatic because a delayed alert may no longer describe the current price.
+model WebhookDeliveryFailure {
+  id             String   @id @default(uuid())
+  endpoint       String
+  payload        Json
+  attemptCount   Int      @map("attempt_count")
+  lastStatus     Int?     @map("last_status")
+  errorCategory  String   @map("error_category")
+  firstAttemptAt DateTime @map("first_attempt_at")
+  lastAttemptAt  DateTime @map("last_attempt_at")
+  createdAt      DateTime @default(now()) @map("created_at")
+
+  @@index([createdAt(sort: Desc)])
+  @@map("webhook_delivery_failures")
+}
+
 model ApiKey {
   id               String    @id @default(uuid())
   hash             String    @unique

src/__tests__/auth.test.ts

@@ -11,6 +11,11 @@ vi.mock('../db', () => ({
       create: vi.fn(),
       update: vi.fn(),
     },
+    webhookDeliveryFailure: {
+      findMany: vi.fn(),
+      create: vi.fn(),
+      deleteMany: vi.fn(),
+    },
   },
 }))
 
@@ -347,6 +352,30 @@ describe('admin endpoints', () => {
     expect(res.statusCode).toBe(404)
   })
 
+  it('protects the recent webhook failure endpoint with the admin token', async () => {
+    const app = await buildAdminApp()
+    const res = await app.inject({ method: 'GET', url: '/admin/webhook-delivery-failures' })
+    expect(res.statusCode).toBe(401)
+    expect(prisma.webhookDeliveryFailure.findMany).not.toHaveBeenCalled()
+  })
+
+  it('returns a bounded newest-first diagnostic list to admins', async () => {
+    const failures = [{ id: 'failed-1', endpoint: 'https://hooks.example.test/alerts', errorCategory: 'http_5xx' }]
+    vi.mocked(prisma.webhookDeliveryFailure.findMany).mockResolvedValue(failures as any)
+    const app = await buildAdminApp()
+    const res = await app.inject({
+      method: 'GET',
+      url: '/admin/webhook-delivery-failures?limit=150',
+      headers: { 'x-admin-token': 'admin-secret' },
+    })
+    expect(res.statusCode).toBe(200)
+    expect(res.json()).toEqual({ failures })
+    expect(prisma.webhookDeliveryFailure.findMany).toHaveBeenCalledWith({
+      orderBy: { createdAt: 'desc' },
+      take: 100,
+    })
+  })
+
   it('registers authentication counters in the Prometheus registry', async () => {
     const output = await register.metrics()
     expect(output).toContain('api_key_auth_failures_total')

src/__tests__/deliveryFailures.test.ts

@@ -0,0 +1,66 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { createFailure, incrementFailureMetric } = vi.hoisted(() => ({
+  createFailure: vi.fn(),
+  incrementFailureMetric: vi.fn(),
+}))
+
+vi.mock('../db', () => ({
+  prisma: { webhookDeliveryFailure: { create: createFailure, deleteMany: vi.fn() } },
+}))
+vi.mock('../metrics', () => ({
+  webhook_delivery_failures_total: { inc: incrementFailureMetric },
+}))
+
+import { prisma } from '../db'
+import { webhook_delivery_failures_total } from '../metrics'
+import { deliverJsonWithRetries } from '../alerts/delivery'
+
+describe('permanent webhook delivery failures', () => {
+  beforeEach(() => {
+    vi.restoreAllMocks()
+    createFailure.mockReset().mockResolvedValue({})
+    incrementFailureMetric.mockReset()
+    global.fetch = vi.fn()
+  })
+
+  it('persists an exhausted response without storing the delivery secret', async () => {
+    vi.mocked(global.fetch).mockResolvedValue({ ok: false, status: 503 } as Response)
+    const firstAttemptAt = new Date('2026-10-16T10:00:00.000Z')
+
+    await deliverJsonWithRetries(
+      'https://user:password@hooks.example.test/alerts?token=private',
+      { assetA: 'XLM', price: 0.123 },
+      'webhook-secret',
+      3,
+      firstAttemptAt,
+    )
+
+    expect(prisma.webhookDeliveryFailure.create).toHaveBeenCalledOnce()
+    const data = vi.mocked(prisma.webhookDeliveryFailure.create).mock.calls[0][0].data
+    expect(data).toMatchObject({
+      endpoint: 'https://hooks.example.test',
+      payload: { assetA: 'XLM', price: 0.123 },
+      attemptCount: 3,
+      lastStatus: 503,
+      errorCategory: 'http_5xx',
+      firstAttemptAt,
+    })
+    expect(data).not.toHaveProperty('secret')
+    expect(data.lastAttemptAt).toBeInstanceOf(Date)
+    expect(JSON.stringify(data)).not.toContain('webhook-secret')
+    expect(JSON.stringify(data)).not.toContain('private')
+    expect(webhook_delivery_failures_total.inc).toHaveBeenCalledWith({ category: 'http_5xx' })
+  })
+
+  it('records a non-retryable client response as a permanent failure', async () => {
+    vi.mocked(global.fetch).mockResolvedValue({ ok: false, status: 410 } as Response)
+
+    await deliverJsonWithRetries('https://hooks.example.test/alerts', { event: 'threshold' }, 'secret')
+
+    expect(prisma.webhookDeliveryFailure.create).toHaveBeenCalledOnce()
+    expect(vi.mocked(prisma.webhookDeliveryFailure.create).mock.calls[0][0].data)
+      .toMatchObject({ attemptCount: 1, lastStatus: 410, errorCategory: 'http_4xx' })
+    expect(webhook_delivery_failures_total.inc).toHaveBeenCalledWith({ category: 'http_4xx' })
+  })
+})

src/__tests__/webhooks.test.ts

@@ -10,6 +10,11 @@ vi.mock('../db', () => ({
       delete: vi.fn(),
       findMany: vi.fn(),
     },
+    webhookDeliveryFailure: {
+      create: vi.fn(),
+      deleteMany: vi.fn(),
+      findMany: vi.fn(),
+    },
   },
 }))
 
@@ -303,4 +308,4 @@ describe('dispatchPriceUpdate', () => {
       },
     })
   })
-})
+})

src/alerts/delivery.ts

@@ -1,8 +1,52 @@
 import { createHmac } from 'crypto'
 import { createLogger } from '../logger'
+import { fetchWithTimeout } from '../fetch'
+import { prisma } from '../db'
+import { webhook_delivery_failures_total } from '../metrics'
 
 const log = createLogger('webhook')
-import { fetchWithTimeout } from '../fetch'
+
+type FailureCategory = 'network' | 'timeout' | 'http_4xx' | 'http_5xx'
+
+function safeEndpoint(raw: string): string {
+  try {
+    const url = new URL(raw)
+    // Any URL component may contain a subscriber secret (for example Slack
+    // webhook paths), so only retain the host for diagnosis.
+    return url.origin
+  } catch {
+    return '[invalid endpoint]'
+  }
+}
+
+async function recordPermanentFailure(args: {
+  url: string
+  payload: string
+  attempt: number
+  status?: number
+  category: FailureCategory
+  firstAttemptAt: Date
+}): Promise<void> {
+  webhook_delivery_failures_total.inc({ category: args.category })
+  try {
+    await prisma.webhookDeliveryFailure.create({
+      data: {
+        endpoint: safeEndpoint(args.url),
+        payload: JSON.parse(args.payload),
+        attemptCount: args.attempt,
+        lastStatus: args.status ?? null,
+        errorCategory: args.category,
+        firstAttemptAt: args.firstAttemptAt,
+        lastAttemptAt: new Date(),
+      },
+    })
+    await prisma.webhookDeliveryFailure.deleteMany({
+      where: { createdAt: { lt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000) } },
+    })
+  } catch (err) {
+    console.error('[webhook] could not persist permanent delivery failure:', (err as Error).message)
+  }
+}
 
 function retryDelayMs(attempt: number): number {
   return Math.pow(2, attempt - 1) * 1000
@@ -17,6 +61,7 @@ export async function deliverJsonWithRetries(
   body: object,
   secret: string,
   attempt = 1,
+  firstAttemptAt = new Date(),
 ): Promise<void> {
   const payload = JSON.stringify(body)
   const signature = signPayload(payload, secret)
@@ -35,24 +80,30 @@ export async function deliverJsonWithRetries(
   } catch (err) {
     if (attempt < 3) {
       await new Promise(resolve => setTimeout(resolve, retryDelayMs(attempt)))
-      return deliverJsonWithRetries(url, body, secret, attempt + 1)
+      return deliverJsonWithRetries(url, body, secret, attempt + 1, firstAttemptAt)
     }
 
-    log.warn({ url, attempt, err }, 'webhook delivery failed after retries')
+    const failureCategory: FailureCategory = ['AbortError', 'TimeoutError'].includes((err as Error)?.name)
+      ? 'timeout'
+      : 'network'
+    await recordPermanentFailure({ url, payload, attempt, category: failureCategory, firstAttemptAt })
+    log.warn({ endpoint: safeEndpoint(url), attempt, category: failureCategory, err }, 'webhook delivery failed after retries')
     return
   }
 
   if (res.ok) return
 
   if (res.status >= 400 && res.status < 500) {
-    log.warn({ url, status: res.status }, 'webhook client error; not retrying')
+    await recordPermanentFailure({ url, payload, attempt, status: res.status, category: 'http_4xx', firstAttemptAt })
+    log.warn({ endpoint: safeEndpoint(url), status: res.status }, 'webhook client error; not retrying')
     return
   }
 
   if (attempt < 3) {
     await new Promise(resolve => setTimeout(resolve, retryDelayMs(attempt)))
-    return deliverJsonWithRetries(url, body, secret, attempt + 1)
+    return deliverJsonWithRetries(url, body, secret, attempt + 1, firstAttemptAt)
   }
 
-  log.warn({ url, attempt, status: res.status }, 'webhook delivery failed after retries')
-}
+  await recordPermanentFailure({ url, payload, attempt, status: res.status, category: 'http_5xx', firstAttemptAt })
+  log.warn({ endpoint: safeEndpoint(url), attempt, status: res.status }, 'webhook delivery failed after retries')
+}

src/api/admin.ts

@@ -29,6 +29,25 @@ interface CreateKeyBody {
  * them; they enforce their own `ADMIN_TOKEN` check instead.
  */
 export async function registerAdminRoutes(app: FastifyInstance) {
+  app.get<{ Querystring: { limit?: string } }>(
+    '/admin/webhook-delivery-failures',
+    { config: { public: true } },
+    async (req, reply) => {
+      if (!isAdminAuthorized(req)) {
+        return reply.status(401).send({ error: 'Unauthorized', message: 'Provide a valid X-Admin-Token header.' })
+      }
+      const requestedLimit = Number(req.query.limit ?? 25)
+      if (!Number.isInteger(requestedLimit) || requestedLimit < 1) {
+        return reply.status(400).send({ error: 'limit must be a positive integer' })
+      }
+      const failures = await prisma.webhookDeliveryFailure.findMany({
+        orderBy: { createdAt: 'desc' },
+        take: Math.min(requestedLimit, 100),
+      })
+      return reply.send({ failures })
+    },
+  )
+
   app.post<{ Body: CreateKeyBody }>(
     '/admin/keys',
     { config: { public: true } },

src/metrics.ts

@@ -78,6 +78,13 @@ export const x402_payments_received_total = new Counter({
   registers: [register]
 })
 
+export const webhook_delivery_failures_total = new Counter({
+  name: 'webhook_delivery_failures_total',
+  help: 'Permanent webhook delivery failures by bounded failure category',
+  labelNames: ['category'],
+  registers: [register]
+})
+
 export const last_trade_timestamp = new Gauge({
   name: 'last_trade_timestamp',
   help: 'Unix timestamp of the last trade ingested for a pair',

src/openapi/coverage.ts

@@ -42,6 +42,7 @@ export const INTERNAL_ROUTES: readonly string[] = [
   // Operator-only usage/quota reporting for minted keys.
   '/admin/usage',
   '/admin/usage/{keyId}',
+  '/admin/webhook-delivery-failures',
 ] as const
 
 /**