diff --git a/README.md b/README.md index 05af2cf3..1ebda442 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,13 @@ Aggregates price data from Stellar's Classic Order Book (SDEX) and AMM Liquidity | POST | `/settle` | Settle an x402 payment (idempotent — retries replay the stored result) | | POST | `/webhooks` | Subscribe to a price-threshold webhook | | DELETE | `/webhooks/:id` | Delete a webhook subscription | +| GET | `/admin/webhook-delivery-failures?limit=25` | Recent permanent webhook failures (requires `X-Admin-Token`) | | GET/POST | `/graphql` | GraphQL query/mutation endpoint (GraphiQL IDE at `/graphiql`, docs: [`docs/graphql.md`](docs/graphql.md)) | Every non-internal route above has a matching entry in [`openapi.yaml`](openapi.yaml); `tests/openapi.test.ts` boots the server and fails if a registered route is missing from the spec. Operator-only and non-HTTP routes (`/admin/keys*`, -`/admin/usage*`, `/metrics`, `/ws`, `/graphiql*`) are deliberately excluded via +`/admin/usage*`, `/admin/webhook-delivery-failures`, `/metrics`, `/ws`, `/graphiql*`) are deliberately excluded via the commented allow-list in [`src/openapi/coverage.ts`](src/openapi/coverage.ts). The `?network=testnet\|mainnet` query param (or `x-network` header) selects the diff --git a/docs/cookbook/alert-bot.md b/docs/cookbook/alert-bot.md index 3fe9001d..25ab77e8 100644 --- a/docs/cookbook/alert-bot.md +++ b/docs/cookbook/alert-bot.md @@ -183,6 +183,15 @@ Delete it with `DELETE /webhooks/:id`. Use the **bot** when you want local control/custom logic, and **webhooks** when you want Lens to hold the subscription. +If delivery permanently fails, Lens stores the endpoint host, alert +payload, attempt count, final HTTP status (when available), failure category, +and first/last attempt times. URL credentials, paths, query parameters, +fragments, and the HMAC secret are excluded from the stored endpoint/record. Recent failures are +available to operators at `GET /admin/webhook-delivery-failures` with an +`X-Admin-Token` header; `limit` defaults to 25 and is capped at 100. Records are +retained for 30 days. They are diagnostic only and are not replayed +automatically, since a delayed threshold alert can be stale. + ## Live demo - **Interactive API explorer (GraphiQL):** run Lens locally and open diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6f8838c4..3dd29988 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -256,6 +256,23 @@ model Webhook { @@map("webhooks") } +/// Permanent webhook failures are retained for 30-day diagnosis only; replay is +/// not automatic because a delayed alert may no longer describe the current price. +model WebhookDeliveryFailure { + id String @id @default(uuid()) + endpoint String + payload Json + attemptCount Int @map("attempt_count") + lastStatus Int? @map("last_status") + errorCategory String @map("error_category") + firstAttemptAt DateTime @map("first_attempt_at") + lastAttemptAt DateTime @map("last_attempt_at") + createdAt DateTime @default(now()) @map("created_at") + + @@index([createdAt(sort: Desc)]) + @@map("webhook_delivery_failures") +} + model ApiKey { id String @id @default(uuid()) hash String @unique diff --git a/src/__tests__/auth.test.ts b/src/__tests__/auth.test.ts index 77915164..12e8ce70 100644 --- a/src/__tests__/auth.test.ts +++ b/src/__tests__/auth.test.ts @@ -11,6 +11,11 @@ vi.mock('../db', () => ({ create: vi.fn(), update: vi.fn(), }, + webhookDeliveryFailure: { + findMany: vi.fn(), + create: vi.fn(), + deleteMany: vi.fn(), + }, }, })) @@ -347,6 +352,30 @@ describe('admin endpoints', () => { expect(res.statusCode).toBe(404) }) + it('protects the recent webhook failure endpoint with the admin token', async () => { + const app = await buildAdminApp() + const res = await app.inject({ method: 'GET', url: '/admin/webhook-delivery-failures' }) + expect(res.statusCode).toBe(401) + expect(prisma.webhookDeliveryFailure.findMany).not.toHaveBeenCalled() + }) + + it('returns a bounded newest-first diagnostic list to admins', async () => { + const failures = [{ id: 'failed-1', endpoint: 'https://hooks.example.test/alerts', errorCategory: 'http_5xx' }] + vi.mocked(prisma.webhookDeliveryFailure.findMany).mockResolvedValue(failures as any) + const app = await buildAdminApp() + const res = await app.inject({ + method: 'GET', + url: '/admin/webhook-delivery-failures?limit=150', + headers: { 'x-admin-token': 'admin-secret' }, + }) + expect(res.statusCode).toBe(200) + expect(res.json()).toEqual({ failures }) + expect(prisma.webhookDeliveryFailure.findMany).toHaveBeenCalledWith({ + orderBy: { createdAt: 'desc' }, + take: 100, + }) + }) + it('registers authentication counters in the Prometheus registry', async () => { const output = await register.metrics() expect(output).toContain('api_key_auth_failures_total') diff --git a/src/__tests__/deliveryFailures.test.ts b/src/__tests__/deliveryFailures.test.ts new file mode 100644 index 00000000..3cdd86f1 --- /dev/null +++ b/src/__tests__/deliveryFailures.test.ts @@ -0,0 +1,66 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { createFailure, incrementFailureMetric } = vi.hoisted(() => ({ + createFailure: vi.fn(), + incrementFailureMetric: vi.fn(), +})) + +vi.mock('../db', () => ({ + prisma: { webhookDeliveryFailure: { create: createFailure, deleteMany: vi.fn() } }, +})) +vi.mock('../metrics', () => ({ + webhook_delivery_failures_total: { inc: incrementFailureMetric }, +})) + +import { prisma } from '../db' +import { webhook_delivery_failures_total } from '../metrics' +import { deliverJsonWithRetries } from '../alerts/delivery' + +describe('permanent webhook delivery failures', () => { + beforeEach(() => { + vi.restoreAllMocks() + createFailure.mockReset().mockResolvedValue({}) + incrementFailureMetric.mockReset() + global.fetch = vi.fn() + }) + + it('persists an exhausted response without storing the delivery secret', async () => { + vi.mocked(global.fetch).mockResolvedValue({ ok: false, status: 503 } as Response) + const firstAttemptAt = new Date('2026-10-16T10:00:00.000Z') + + await deliverJsonWithRetries( + 'https://user:password@hooks.example.test/alerts?token=private', + { assetA: 'XLM', price: 0.123 }, + 'webhook-secret', + 3, + firstAttemptAt, + ) + + expect(prisma.webhookDeliveryFailure.create).toHaveBeenCalledOnce() + const data = vi.mocked(prisma.webhookDeliveryFailure.create).mock.calls[0][0].data + expect(data).toMatchObject({ + endpoint: 'https://hooks.example.test', + payload: { assetA: 'XLM', price: 0.123 }, + attemptCount: 3, + lastStatus: 503, + errorCategory: 'http_5xx', + firstAttemptAt, + }) + expect(data).not.toHaveProperty('secret') + expect(data.lastAttemptAt).toBeInstanceOf(Date) + expect(JSON.stringify(data)).not.toContain('webhook-secret') + expect(JSON.stringify(data)).not.toContain('private') + expect(webhook_delivery_failures_total.inc).toHaveBeenCalledWith({ category: 'http_5xx' }) + }) + + it('records a non-retryable client response as a permanent failure', async () => { + vi.mocked(global.fetch).mockResolvedValue({ ok: false, status: 410 } as Response) + + await deliverJsonWithRetries('https://hooks.example.test/alerts', { event: 'threshold' }, 'secret') + + expect(prisma.webhookDeliveryFailure.create).toHaveBeenCalledOnce() + expect(vi.mocked(prisma.webhookDeliveryFailure.create).mock.calls[0][0].data) + .toMatchObject({ attemptCount: 1, lastStatus: 410, errorCategory: 'http_4xx' }) + expect(webhook_delivery_failures_total.inc).toHaveBeenCalledWith({ category: 'http_4xx' }) + }) +}) diff --git a/src/__tests__/webhooks.test.ts b/src/__tests__/webhooks.test.ts index e40bea49..69806554 100644 --- a/src/__tests__/webhooks.test.ts +++ b/src/__tests__/webhooks.test.ts @@ -10,6 +10,11 @@ vi.mock('../db', () => ({ delete: vi.fn(), findMany: vi.fn(), }, + webhookDeliveryFailure: { + create: vi.fn(), + deleteMany: vi.fn(), + findMany: vi.fn(), + }, }, })) @@ -303,4 +308,4 @@ describe('dispatchPriceUpdate', () => { }, }) }) -}) \ No newline at end of file +}) diff --git a/src/alerts/delivery.ts b/src/alerts/delivery.ts index 1c0b28af..cd2a7fbb 100644 --- a/src/alerts/delivery.ts +++ b/src/alerts/delivery.ts @@ -1,8 +1,52 @@ import { createHmac } from 'crypto' import { createLogger } from '../logger' +import { fetchWithTimeout } from '../fetch' +import { prisma } from '../db' +import { webhook_delivery_failures_total } from '../metrics' const log = createLogger('webhook') -import { fetchWithTimeout } from '../fetch' + +type FailureCategory = 'network' | 'timeout' | 'http_4xx' | 'http_5xx' + +function safeEndpoint(raw: string): string { + try { + const url = new URL(raw) + // Any URL component may contain a subscriber secret (for example Slack + // webhook paths), so only retain the host for diagnosis. + return url.origin + } catch { + return '[invalid endpoint]' + } +} + +async function recordPermanentFailure(args: { + url: string + payload: string + attempt: number + status?: number + category: FailureCategory + firstAttemptAt: Date +}): Promise { + webhook_delivery_failures_total.inc({ category: args.category }) + try { + await prisma.webhookDeliveryFailure.create({ + data: { + endpoint: safeEndpoint(args.url), + payload: JSON.parse(args.payload), + attemptCount: args.attempt, + lastStatus: args.status ?? null, + errorCategory: args.category, + firstAttemptAt: args.firstAttemptAt, + lastAttemptAt: new Date(), + }, + }) + await prisma.webhookDeliveryFailure.deleteMany({ + where: { createdAt: { lt: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000) } }, + }) + } catch (err) { + console.error('[webhook] could not persist permanent delivery failure:', (err as Error).message) + } +} function retryDelayMs(attempt: number): number { return Math.pow(2, attempt - 1) * 1000 @@ -17,6 +61,7 @@ export async function deliverJsonWithRetries( body: object, secret: string, attempt = 1, + firstAttemptAt = new Date(), ): Promise { const payload = JSON.stringify(body) const signature = signPayload(payload, secret) @@ -35,24 +80,30 @@ export async function deliverJsonWithRetries( } catch (err) { if (attempt < 3) { await new Promise(resolve => setTimeout(resolve, retryDelayMs(attempt))) - return deliverJsonWithRetries(url, body, secret, attempt + 1) + return deliverJsonWithRetries(url, body, secret, attempt + 1, firstAttemptAt) } - log.warn({ url, attempt, err }, 'webhook delivery failed after retries') + const failureCategory: FailureCategory = ['AbortError', 'TimeoutError'].includes((err as Error)?.name) + ? 'timeout' + : 'network' + await recordPermanentFailure({ url, payload, attempt, category: failureCategory, firstAttemptAt }) + log.warn({ endpoint: safeEndpoint(url), attempt, category: failureCategory, err }, 'webhook delivery failed after retries') return } if (res.ok) return if (res.status >= 400 && res.status < 500) { - log.warn({ url, status: res.status }, 'webhook client error; not retrying') + await recordPermanentFailure({ url, payload, attempt, status: res.status, category: 'http_4xx', firstAttemptAt }) + log.warn({ endpoint: safeEndpoint(url), status: res.status }, 'webhook client error; not retrying') return } if (attempt < 3) { await new Promise(resolve => setTimeout(resolve, retryDelayMs(attempt))) - return deliverJsonWithRetries(url, body, secret, attempt + 1) + return deliverJsonWithRetries(url, body, secret, attempt + 1, firstAttemptAt) } - log.warn({ url, attempt, status: res.status }, 'webhook delivery failed after retries') -} \ No newline at end of file + await recordPermanentFailure({ url, payload, attempt, status: res.status, category: 'http_5xx', firstAttemptAt }) + log.warn({ endpoint: safeEndpoint(url), attempt, status: res.status }, 'webhook delivery failed after retries') +} diff --git a/src/api/admin.ts b/src/api/admin.ts index 6cd01f68..62e0a604 100644 --- a/src/api/admin.ts +++ b/src/api/admin.ts @@ -29,6 +29,25 @@ interface CreateKeyBody { * them; they enforce their own `ADMIN_TOKEN` check instead. */ export async function registerAdminRoutes(app: FastifyInstance) { + app.get<{ Querystring: { limit?: string } }>( + '/admin/webhook-delivery-failures', + { config: { public: true } }, + async (req, reply) => { + if (!isAdminAuthorized(req)) { + return reply.status(401).send({ error: 'Unauthorized', message: 'Provide a valid X-Admin-Token header.' }) + } + const requestedLimit = Number(req.query.limit ?? 25) + if (!Number.isInteger(requestedLimit) || requestedLimit < 1) { + return reply.status(400).send({ error: 'limit must be a positive integer' }) + } + const failures = await prisma.webhookDeliveryFailure.findMany({ + orderBy: { createdAt: 'desc' }, + take: Math.min(requestedLimit, 100), + }) + return reply.send({ failures }) + }, + ) + app.post<{ Body: CreateKeyBody }>( '/admin/keys', { config: { public: true } }, diff --git a/src/metrics.ts b/src/metrics.ts index ac8ac646..cee185f3 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -78,6 +78,13 @@ export const x402_payments_received_total = new Counter({ registers: [register] }) +export const webhook_delivery_failures_total = new Counter({ + name: 'webhook_delivery_failures_total', + help: 'Permanent webhook delivery failures by bounded failure category', + labelNames: ['category'], + registers: [register] +}) + export const last_trade_timestamp = new Gauge({ name: 'last_trade_timestamp', help: 'Unix timestamp of the last trade ingested for a pair', diff --git a/src/openapi/coverage.ts b/src/openapi/coverage.ts index 3e09587f..43c1d2af 100644 --- a/src/openapi/coverage.ts +++ b/src/openapi/coverage.ts @@ -42,6 +42,7 @@ export const INTERNAL_ROUTES: readonly string[] = [ // Operator-only usage/quota reporting for minted keys. '/admin/usage', '/admin/usage/{keyId}', + '/admin/webhook-delivery-failures', ] as const /**