From 3cbcbcd8c1d5f7c06c982a0a50f1ebee9785b93e Mon Sep 17 00:00:00 2001 From: Buffly Date: Tue, 20 Oct 2026 16:52:53 +0000 Subject: [PATCH 1/3] feat(status): separate process health from data health /status was excluded from the OpenAPI contract, so nothing described its shape or which values mean unhealthy. Document the full response, report process health and data health as separate fields (data fresh/stale/empty), and make 503 the single paging condition when a network's ingester falls behind STATUS_STALE_AFTER_SECONDS. Co-Authored-By: Codebuff --- .env.example | 4 + README.md | 14 +- docs/status.md | 162 +++++++++++++++++++++++ openapi.json | 241 ++++++++++++++++++++++++++++++----- openapi.yaml | 218 +++++++++++++++++++++++++++---- src/__tests__/status.test.ts | 59 +++++++++ src/api/rest.ts | 82 +++++++++--- src/api/schemas.ts | 46 ++++++- src/config.ts | 6 + 9 files changed, 752 insertions(+), 80 deletions(-) create mode 100644 docs/status.md diff --git a/.env.example b/.env.example index 41e4465f..17bf8f95 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,10 @@ REDIS_URL=redis://localhost:6379 PRICE_CACHE_TTL=10 # Default timeout for outbound HTTP requests in milliseconds (default: 10000) HTTP_TIMEOUT_MS=10000 +# How old the newest ingester write may be before GET /status reports +# data.status "stale" and answers 503. Keep it well above POLL_INTERVAL_MS. +# Default: 300 seconds. See docs/status.md. +STATUS_STALE_AFTER_SECONDS=300 # --- Active Network --- # Which network the node operates on: "testnet" or "mainnet" (default: testnet) diff --git a/README.md b/README.md index 8351617e..3da06c30 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Aggregates price data from Stellar's Classic Order Book (SDEX) and AMM Liquidity | GET | `/screener?sortBy=volume&order=desc&limit=20` | Screen pairs by volume, 24h change, liquidity and price | | GET | `/benchmark/:asset?target=USD` | Peg-deviation statistics versus a target | | GET | `/basket?asset=XLM&asset=USDC&weight=0.6&weight=0.4` | Weighted price of a basket of assets | -| GET | `/status` | Indexer health | +| GET | `/status` | Process and indexer health ([contract](docs/status.md)) | | GET | `/discovery/resources?type=&payTo=&network=&extensions=&limit=&offset=` | Bazaar catalog of x402-discoverable resources (spec: [`bazaar`](https://github.com/x402-foundation/x402/blob/main/specs/extensions/bazaar.md)) | | GET | `/usage/me` | Usage and quota for the calling API key (requires `Authorization: Bearer `) | | GET | `/supported` | Payment kinds and extensions this facilitator supports | @@ -216,8 +216,10 @@ curl http://localhost:3002/status # "network": "testnet", # "watchedPairs": ["XLM:native/USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"], # "lastIndexedLedger": 53842917, -# "lastProcessedAt": "2026-05-07T18:45:11.220Z", -# "ingestLagSeconds": 4 +# "lastProcessedAt": "2026-10-20T16:45:11.220Z", +# "ingestLagSeconds": 4, +# "process": { "status": "up", "uptimeSeconds": 86400 }, +# "data": { "status": "fresh", "ingestLagSeconds": 4, "staleAfterSeconds": 300 } # } ``` @@ -226,6 +228,12 @@ curl http://localhost:3002/status `null` until that network has ingested, and `lastIndexedLedger` is `null` until its SDEX ingester has recorded a trade. +Process health and data health are separate: `process.status` is `up` whenever +the process answers, while `data.status` is `fresh`, `stale` or `empty`. A +stale ingester answers **`503`** (the only paging condition); `empty` stays +`200`. See [`docs/status.md`](docs/status.md) for the full shape, status-code +contract and monitor configuration examples. + ### 2. Paid request without `X-PAYMENT` → `402` with payment requirements ```bash diff --git a/docs/status.md b/docs/status.md new file mode 100644 index 00000000..9dd5a091 --- /dev/null +++ b/docs/status.md @@ -0,0 +1,162 @@ +# `GET /status` — health contract + +`/status` is the endpoint uptime monitors and integrators hit first. It is +public, needs no API key and is not gated by x402. + +It reports **two independent health verdicts**: + +| Field | Answers | Values | +|---|---|---| +| `process.status` | Is this HTTP process able to answer? | `up` | +| `data.status` | Is the ingester behind the selected network current? | `fresh`, `stale`, `empty` | + +The process can be up while every venue is stale. Those two states must not +look alike, so they are never collapsed into one flag: `ok` and `process` are +about the process, `data` is about the data. + +## Response shape + +```json +{ + "ok": true, + "network": "mainnet", + "watchedPairs": [ + "USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM" + ], + "lastIndexedLedger": 53842917, + "lastProcessedAt": "2026-10-20T16:45:11.220Z", + "ingestLagSeconds": 4, + "process": { + "status": "up", + "uptimeSeconds": 86400 + }, + "data": { + "status": "fresh", + "ingestLagSeconds": 4, + "staleAfterSeconds": 300 + } +} +``` + +| Field | Type | Meaning | +|---|---|---| +| `ok` | boolean | Process health only. `true` whenever the process serves this response. It says nothing about data freshness — read `data.status` or the HTTP status code for that. | +| `network` | `"testnet"` \| `"mainnet"` | The network this report covers, resolved from `?network=` / `x-network` and defaulting to the instance's `STELLAR_NETWORK`. | +| `watchedPairs` | string[] | Canonical pair keys this network's ingester watches. | +| `lastIndexedLedger` | integer \| null | Newest ledger recorded for this network. `null` until its SDEX ingester records a trade. | +| `lastProcessedAt` | ISO 8601 \| null | Timestamp of the newest ingester write for this network. `null` before the first one. | +| `ingestLagSeconds` | integer \| null | Seconds since `lastProcessedAt`. `null` when the network has never ingested. Top-level mirror of `data.ingestLagSeconds`. | +| `process.status` | `"up"` | The process is answering. A monitor learns the process is *down* from a failed connection, not from a body field. | +| `process.uptimeSeconds` | integer | Seconds since this process started. | +| `data.status` | `"fresh"` \| `"stale"` \| `"empty"` | `fresh` when `ingestLagSeconds <= staleAfterSeconds`; `stale` when it is greater; `empty` when the network has never ingested. | +| `data.ingestLagSeconds` | integer \| null | Same value as the top-level field, grouped with the verdict that uses it. | +| `data.staleAfterSeconds` | integer | The staleness budget in force (`STATUS_STALE_AFTER_SECONDS`, default `300`). | + +`lastIndexedLedger`, `lastProcessedAt` and `ingestLagSeconds` predate the +process/data split and are kept at the top level so existing consumers keep +working; new integrations can read everything from `data`. + +### `empty` is not `stale` + +A network that has never ingested has no lag to measure. Reporting it as stale +would page on every fresh deployment, so it is `empty` and answered with `200`. +A monitor that must know an ingester started can alert on +`data.status == "empty"` after its own startup grace period. + +## HTTP status codes + +| Code | When | Page? | +|---|---|---| +| `200` | `data.status` is `fresh` or `empty` | No | +| `503` | `data.status` is `stale` — newest write is older than `staleAfterSeconds` | **Yes** | +| `500` | Indexer state could not be read, so no health report could be produced | Yes | +| `400` | Unrecognised `?network=` value (only `testnet` / `mainnet`) | No — fix the monitor | + +`/status` is exempt from the API rate limiter, so it never answers `429`. A +monitor polling every few seconds will not exhaust a quota. + +**The paging condition is `503`.** It is the only status this endpoint returns +for "up, but its data is too old": the process answered, so the request itself +succeeded, but every price behind it is older than the staleness budget. A +monitor configuration that pages on any non-2xx response and treats `400` as a +configuration problem is correct. + +`503` responses carry the full status body (including `data.ingestLagSeconds` +and `Retry-After: 60`), so an alert can say *how far* behind the ingester is +instead of only that it is behind. + +## Tuning the staleness budget + +```bash +# Seconds the newest ingester write may age before /status reports stale. +# Default: 300 (5 minutes). +STATUS_STALE_AFTER_SECONDS=300 +``` + +Keep this comfortably above the ingester's poll interval (`POLL_INTERVAL_MS`, +default `5000`). A value near the poll interval pages on a single missed poll. + +## Monitor configuration examples + +### UptimeRobot (free plan) + +1. **Add New Monitor** → **HTTP(s)**. +2. URL: `https:///status?network=mainnet` — one monitor per + network you index. +3. Interval: 5 minutes. +4. Under **Alert Contacts**, tick your contact. Leave the advanced settings at + their defaults: UptimeRobot marks the monitor down on any non-2xx response, + which is exactly the paging condition above (`503` for stale data, `500` for + an unreadable health report). + +If you would rather alert only when the data is stale, use **Keyword** +monitoring instead: the keyword `"status":"stale"` must **not** exist in the +response body (the response is minified JSON, so match it without spaces). + +### Shell / cron (curl + jq) + +```bash +#!/usr/bin/env bash +set -euo pipefail + +# -f turns 503 (stale) and 500 into a non-zero exit code. +curl -fsS --max-time 5 \ + 'https:///status?network=mainnet' | + jq -e '.process.status == "up"' > /dev/null +``` + +Exit codes: `0` fresh, `22` stale (HTTP 503), `28` timeout/unreachable. Wire +the non-zero exit into your existing alerting. + +For "ingester never started" as well: + +```bash +curl -fsS --max-time 5 'https:///status?network=mainnet' | + jq -e '.data.status == "fresh"' +``` + +### Prometheus `blackbox_exporter` + +```yaml +modules: + lens_status: + prober: http + timeout: 5s + http: + valid_status_codes: [200] # 503 = stale data, alerts + preferred_ip_protocol: ip4 + fail_if_body_matches_regexp: + - '"status":"stale"' +``` + +Alert on `probe_success == 0` for 2 consecutive scrapes (10 minutes at a +`scrape_interval` of 5m) to ignore a single missed poll. + +### Kubernetes + +- **Liveness probe:** process health only. Use `httpGet: /status` and rely on + the connection succeeding — do not fail liveness on data staleness, or a + stalled Horizon will restart a healthy process in a loop. +- **Readiness probe:** if your consumers must never see stale prices, point it + at `/status` as well; `503` then removes the pod from the load balancer until + the ingester catches up. diff --git a/openapi.json b/openapi.json index cb6441b7..1443cccb 100644 --- a/openapi.json +++ b/openapi.json @@ -19,7 +19,8 @@ "/status": { "get": { "operationId": "getStatus", - "summary": "Get API status and indexer state", + "summary": "Get process health and indexer (data) health", + "description": "Public, unauthenticated health endpoint for uptime monitors and integrators. See [docs/status.md](https://github.com/Divinity-Labs/Lens/blob/main/docs/status.md) for the monitor configuration examples.\n\nProcess health and data health are reported separately. A process can be up while every venue's data is stale, and the two must not look alike:\n* `process.status` is `up` whenever this process answers the request.\n* `data.status` classifies the newest ingester write for the selected network as `fresh` (within `staleAfterSeconds`), `stale` (older than it), or `empty` (that network has not ingested yet).\n\nOnly `503` is a paging condition: it means the process is up but the selected network's data is stale. `200` covers `fresh` and `empty`. `400` (invalid `network`) is a monitor misconfiguration and `500` (unexpected error) means no report could be produced; neither is a staleness signal. `/status` is exempt from the API rate limiter, so it never answers 429.", "security": [], "parameters": [ { @@ -28,48 +29,120 @@ ], "responses": { "200": { - "description": "API status", + "description": "The process is up and the selected network's data is `fresh` or `empty` (never ingested, so there is no lag to judge).", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + }, + "examples": { + "fresh": { + "summary": "Ingester is current", + "value": { + "ok": true, + "network": "mainnet", + "watchedPairs": [ + "USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM" + ], + "lastIndexedLedger": 53842917, + "lastProcessedAt": "2026-10-20T16:45:11.220Z", + "ingestLagSeconds": 4, + "process": { + "status": "up", + "uptimeSeconds": 86400 + }, + "data": { + "status": "fresh", + "ingestLagSeconds": 4, + "staleAfterSeconds": 300 + } + } + }, + "empty": { + "summary": "Network has never ingested", + "value": { + "ok": true, + "network": "testnet", + "watchedPairs": [], + "lastIndexedLedger": null, + "lastProcessedAt": null, + "ingestLagSeconds": null, + "process": { + "status": "up", + "uptimeSeconds": 12 + }, + "data": { + "status": "empty", + "ingestLagSeconds": null, + "staleAfterSeconds": 300 + } + } + } + } + } + } + }, + "400": { + "description": "Unrecognised `network` value. Not a health signal — the monitor is misconfigured.", "content": { "application/json": { "schema": { "type": "object", + "required": [ + "error" + ], "properties": { - "ok": { - "type": "boolean" - }, - "network": { - "type": "string", - "enum": [ - "testnet", - "mainnet" - ] - }, - "watchedPairs": { - "type": "array", - "items": { - "type": "string" - } - }, - "lastIndexedLedger": { - "type": "integer", - "nullable": true - }, - "lastProcessedAt": { - "type": "string", - "format": "date-time", - "nullable": true - }, - "ingestLagSeconds": { - "type": "integer", - "nullable": true + "error": { + "type": "string" } } + }, + "example": { + "error": "Invalid network \"pubnet\" — expected one of: testnet, mainnet" } } } }, - "429": { - "$ref": "#/components/responses/TooManyRequests" + "500": { + "description": "Unexpected error while reading indexer state. The process could not produce its health report, so the monitor should page." + }, + "503": { + "description": "Paging condition: the process is up but the selected network's data is `stale` — its newest write is older than `data.staleAfterSeconds`. The body is the full status shape, so a monitor can report how far behind the ingester is.", + "headers": { + "Retry-After": { + "description": "Seconds to wait before polling again.", + "schema": { + "type": "integer", + "example": 60 + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + }, + "example": { + "ok": true, + "network": "mainnet", + "watchedPairs": [ + "USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM" + ], + "lastIndexedLedger": 53800000, + "lastProcessedAt": "2026-10-20T16:05:11.220Z", + "ingestLagSeconds": 2404, + "process": { + "status": "up", + "uptimeSeconds": 86400 + }, + "data": { + "status": "stale", + "ingestLagSeconds": 2404, + "staleAfterSeconds": 300 + } + } + } + } } } } @@ -3198,6 +3271,110 @@ } } }, + "StatusResponse": { + "type": "object", + "description": "Process health and data health for one network. `ok` and `process` describe this HTTP process; `data` describes the ingester behind the selected network. The flat `lastIndexedLedger`, `lastProcessedAt` and `ingestLagSeconds` fields mirror the `data` values for older clients.", + "required": [ + "ok", + "network", + "watchedPairs", + "lastIndexedLedger", + "lastProcessedAt", + "ingestLagSeconds", + "process", + "data" + ], + "additionalProperties": false, + "properties": { + "ok": { + "type": "boolean", + "description": "Process health only — true whenever this process can serve the request. It says nothing about whether the data is current; read `data.status` (or the HTTP status code) for that." + }, + "network": { + "type": "string", + "enum": [ + "testnet", + "mainnet" + ], + "description": "The network this report covers." + }, + "watchedPairs": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Canonical pair keys this network's ingester watches." + }, + "lastIndexedLedger": { + "type": "integer", + "nullable": true, + "description": "Newest ledger recorded for this network, or null before its SDEX ingester records a trade. Mirror of the data behind `data.status`." + }, + "lastProcessedAt": { + "type": "string", + "format": "date-time", + "nullable": true, + "description": "Timestamp of the newest ingester write for this network, or null before the first one." + }, + "ingestLagSeconds": { + "type": "integer", + "nullable": true, + "description": "Seconds since `lastProcessedAt`, or null before the first write. Kept as a top-level mirror of `data.ingestLagSeconds`." + }, + "process": { + "type": "object", + "description": "Health of the HTTP process itself.", + "required": [ + "status", + "uptimeSeconds" + ], + "additionalProperties": false, + "properties": { + "status": { + "type": "string", + "enum": [ + "up" + ], + "description": "`up` whenever the process answers. A non-200 response is only produced for the paging condition, never because the process has stopped answering (that is a connection failure)." + }, + "uptimeSeconds": { + "type": "integer", + "description": "Seconds since this process started." + } + } + }, + "data": { + "type": "object", + "description": "Freshness of the ingester data behind the selected network. This is independent of `process.status`: data can be stale while the process is up.", + "required": [ + "status", + "ingestLagSeconds", + "staleAfterSeconds" + ], + "additionalProperties": false, + "properties": { + "status": { + "type": "string", + "enum": [ + "fresh", + "stale", + "empty" + ], + "description": "`fresh` when `ingestLagSeconds` is at most `staleAfterSeconds`; `stale` when it is greater (the only paging condition, answered with 503); `empty` when the network has never ingested, so no lag exists yet (answered with 200)." + }, + "ingestLagSeconds": { + "type": "integer", + "nullable": true, + "description": "Seconds since the newest ingester write, or null when `status` is `empty`." + }, + "staleAfterSeconds": { + "type": "integer", + "description": "The configured staleness budget (`STATUS_STALE_AFTER_SECONDS`, default 300). A lag greater than this makes `status` `stale`." + } + } + } + } + }, "GraphQlResponse": { "type": "object", "additionalProperties": true, diff --git a/openapi.yaml b/openapi.yaml index 9d511d0b..f31a0967 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -15,39 +15,125 @@ paths: /status: get: operationId: getStatus - summary: Get API status and indexer state + summary: Get process health and indexer (data) health + description: >- + Public, unauthenticated health endpoint for uptime monitors and + integrators. See [docs/status.md](https://github.com/Divinity-Labs/Lens/blob/main/docs/status.md) + for the monitor configuration examples. + + + Process health and data health are reported separately. A process can + be up while every venue's data is stale, and the two must not look + alike: + + * `process.status` is `up` whenever this process answers the request. + + * `data.status` classifies the newest ingester write for the selected + network as `fresh` (within `staleAfterSeconds`), `stale` (older than + it), or `empty` (that network has not ingested yet). + + + Only `503` is a paging condition: it means the process is up but the + selected network's data is stale. `200` covers `fresh` and `empty`. + `400` (invalid `network`) is a monitor misconfiguration and `500` + (unexpected error) means no report could be produced; neither is a + staleness signal. `/status` is exempt from the API rate limiter, so it + never answers 429. security: [] parameters: - $ref: '#/components/parameters/NetworkQuery' responses: '200': - description: API status + description: >- + The process is up and the selected network's data is `fresh` or + `empty` (never ingested, so there is no lag to judge). + content: + application/json: + schema: + $ref: '#/components/schemas/StatusResponse' + examples: + fresh: + summary: Ingester is current + value: + ok: true + network: mainnet + watchedPairs: + - USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM + lastIndexedLedger: 53842917 + lastProcessedAt: '2026-10-20T16:45:11.220Z' + ingestLagSeconds: 4 + process: + status: up + uptimeSeconds: 86400 + data: + status: fresh + ingestLagSeconds: 4 + staleAfterSeconds: 300 + empty: + summary: Network has never ingested + value: + ok: true + network: testnet + watchedPairs: [] + lastIndexedLedger: null + lastProcessedAt: null + ingestLagSeconds: null + process: + status: up + uptimeSeconds: 12 + data: + status: empty + ingestLagSeconds: null + staleAfterSeconds: 300 + '400': + description: >- + Unrecognised `network` value. Not a health signal — the monitor is + misconfigured. content: application/json: schema: type: object + required: [error] properties: - ok: - type: boolean - network: + error: type: string - enum: [testnet, mainnet] - watchedPairs: - type: array - items: - type: string - lastIndexedLedger: - type: integer - nullable: true - lastProcessedAt: - type: string - format: date-time - nullable: true - ingestLagSeconds: - type: integer - nullable: true - '429': - $ref: '#/components/responses/TooManyRequests' + example: + error: 'Invalid network "pubnet" — expected one of: testnet, mainnet' + '500': + description: >- + Unexpected error while reading indexer state. The process could not + produce its health report, so the monitor should page. + '503': + description: >- + Paging condition: the process is up but the selected network's + data is `stale` — its newest write is older than + `data.staleAfterSeconds`. The body is the full status shape, so a + monitor can report how far behind the ingester is. + headers: + Retry-After: + description: Seconds to wait before polling again. + schema: + type: integer + example: 60 + content: + application/json: + schema: + $ref: '#/components/schemas/StatusResponse' + example: + ok: true + network: mainnet + watchedPairs: + - USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM + lastIndexedLedger: 53800000 + lastProcessedAt: '2026-10-20T16:05:11.220Z' + ingestLagSeconds: 2404 + process: + status: up + uptimeSeconds: 86400 + data: + status: stale + ingestLagSeconds: 2404 + staleAfterSeconds: 300 /price/{assetA}/{assetB}: get: @@ -1990,6 +2076,94 @@ components: items: type: string + StatusResponse: + type: object + description: >- + Process health and data health for one network. `ok` and `process` + describe this HTTP process; `data` describes the ingester behind the + selected network. The flat `lastIndexedLedger`, `lastProcessedAt` and + `ingestLagSeconds` fields mirror the `data` values for older clients. + required: + [ok, network, watchedPairs, lastIndexedLedger, lastProcessedAt, ingestLagSeconds, process, data] + additionalProperties: false + properties: + ok: + type: boolean + description: >- + Process health only — true whenever this process can serve the + request. It says nothing about whether the data is current; read + `data.status` (or the HTTP status code) for that. + network: + type: string + enum: [testnet, mainnet] + description: The network this report covers. + watchedPairs: + type: array + items: + type: string + description: Canonical pair keys this network's ingester watches. + lastIndexedLedger: + type: integer + nullable: true + description: >- + Newest ledger recorded for this network, or null before its SDEX + ingester records a trade. Mirror of the data behind `data.status`. + lastProcessedAt: + type: string + format: date-time + nullable: true + description: >- + Timestamp of the newest ingester write for this network, or null + before the first one. + ingestLagSeconds: + type: integer + nullable: true + description: >- + Seconds since `lastProcessedAt`, or null before the first write. + Kept as a top-level mirror of `data.ingestLagSeconds`. + process: + type: object + description: Health of the HTTP process itself. + required: [status, uptimeSeconds] + additionalProperties: false + properties: + status: + type: string + enum: [up] + description: >- + `up` whenever the process answers. A non-200 response is only + produced for the paging condition, never because the process + has stopped answering (that is a connection failure). + uptimeSeconds: + type: integer + description: Seconds since this process started. + data: + type: object + description: >- + Freshness of the ingester data behind the selected network. This is + independent of `process.status`: data can be stale while the + process is up. + required: [status, ingestLagSeconds, staleAfterSeconds] + additionalProperties: false + properties: + status: + type: string + enum: [fresh, stale, empty] + description: >- + `fresh` when `ingestLagSeconds` is at most `staleAfterSeconds`; + `stale` when it is greater (the only paging condition, answered + with 503); `empty` when the network has never ingested, so no + lag exists yet (answered with 200). + ingestLagSeconds: + type: integer + nullable: true + description: Seconds since the newest ingester write, or null when `status` is `empty`. + staleAfterSeconds: + type: integer + description: >- + The configured staleness budget (`STATUS_STALE_AFTER_SECONDS`, + default 300). A lag greater than this makes `status` `stale`. + GraphQlResponse: type: object additionalProperties: true diff --git a/src/__tests__/status.test.ts b/src/__tests__/status.test.ts index 065bfe17..18f24ecc 100644 --- a/src/__tests__/status.test.ts +++ b/src/__tests__/status.test.ts @@ -43,6 +43,7 @@ vi.mock('../config', () => ({ config: { pairs: pairsByNetwork.testnet, cache: { priceTtl: 10 }, + status: { staleAfterSeconds: 300 }, }, activeNetwork: 'testnet', getNetworkConfig: (network: string) => ({ @@ -117,6 +118,60 @@ describe('GET /status', () => { expect(body.ingestLagSeconds).toBeLessThan(60) }) + it('reports process health and data health as separate fields', async () => { + const app = await buildApp() + + const res = await app.inject({ method: 'GET', url: '/status?network=mainnet' }) + const body = res.json() + + expect(res.statusCode).toBe(200) + expect(body.ok).toBe(true) + expect(body.process).toEqual({ + status: 'up', + uptimeSeconds: expect.any(Number), + }) + expect(Number.isInteger(body.process.uptimeSeconds)).toBe(true) + expect(body.data).toEqual({ + status: 'fresh', + ingestLagSeconds: body.ingestLagSeconds, + staleAfterSeconds: 300, + }) + }) + + it('pages with 503 only when data is stale, while still reporting the process as up', async () => { + mockQuery.mockResolvedValue({ + rows: [{ last_ledger: 222222, last_processed_at: new Date(Date.now() - 3_600_000) }], + }) + const app = await buildApp() + + const res = await app.inject({ method: 'GET', url: '/status?network=mainnet' }) + const body = res.json() + + expect(res.statusCode).toBe(503) + expect(res.headers['retry-after']).toBe('60') + // The process is healthy; only its data is not. + expect(body.ok).toBe(true) + expect(body.process.status).toBe('up') + expect(body.data.status).toBe('stale') + expect(body.data.ingestLagSeconds).toBeGreaterThan(300) + expect(body.data.staleAfterSeconds).toBe(300) + expect(body.lastIndexedLedger).toBe(222222) + expect(body.lastProcessedAt).not.toBeNull() + }) + + it('treats a lag of exactly staleAfterSeconds as fresh', async () => { + mockQuery.mockResolvedValue({ + rows: [{ last_ledger: 222222, last_processed_at: new Date(Date.now() - 300_000) }], + }) + const app = await buildApp() + + const res = await app.inject({ method: 'GET', url: '/status?network=mainnet' }) + + // Rounded lag can land on 299/300; either way it must not be stale. + expect(res.statusCode).toBe(200) + expect(res.json().data.status).toBe('fresh') + }) + it('returns nulls — never another network\'s data — when a network has not ingested', async () => { mockQuery.mockResolvedValue({ rows: [] }) const app = await buildApp() @@ -124,10 +179,14 @@ describe('GET /status', () => { const res = await app.inject({ method: 'GET', url: '/status?network=mainnet' }) const body = res.json() + expect(res.statusCode).toBe(200) expect(body.network).toBe('mainnet') expect(body.lastIndexedLedger).toBeNull() expect(body.lastProcessedAt).toBeNull() expect(body.ingestLagSeconds).toBeNull() + // A network that has never ingested is not stale — there is no lag to + // measure — so it stays a 200 and is distinguishable via data.status. + expect(body.data.status).toBe('empty') }) it('rejects an unrecognised network before answering', async () => { diff --git a/src/api/rest.ts b/src/api/rest.ts index 2f5fbdce..78d0bb76 100644 --- a/src/api/rest.ts +++ b/src/api/rest.ts @@ -68,32 +68,72 @@ export async function registerRESTRoutes(app: FastifyInstance) { // up the validating serializer. installResponseValidation(app) - // GET /status — public health/monitoring endpoint (no API key required) - app.get('/status', { config: { public: true }, schema: { response: { 200: statusResponseSchema } } }, async (req) => { - const network = req.network ?? activeNetwork - const result = await pgPool.query( - `SELECT last_ledger, last_processed_at - FROM indexer_state - WHERE network = $1 - ORDER BY updated_at DESC - LIMIT 1`, - [network] - ) - const lastProcessedAt = result.rows[0]?.last_processed_at ?? null - return { - ok: true, - network, - watchedPairs: getNetworkConfig(network).pairs.map(p => p.pairKey), - lastIndexedLedger: result.rows[0]?.last_ledger ?? null, - lastProcessedAt, + // GET /status — public health/monitoring endpoint (no API key required). + // + // Process health and data health are reported separately: this process can + // be up while every venue's data is stale, and a monitor must be able to tell + // those apart. The only non-2xx outcome that means "page someone" is 503, + // returned when the selected network's data is stale (see docs/status.md). + app.get( + '/status', + { + config: { public: true }, + schema: { response: { 200: statusResponseSchema, 503: statusResponseSchema } }, + }, + async (req, reply) => { + const network = req.network ?? activeNetwork + const result = await pgPool.query( + `SELECT last_ledger, last_processed_at + FROM indexer_state + WHERE network = $1 + ORDER BY updated_at DESC + LIMIT 1`, + [network] + ) + const lastProcessedAt = result.rows[0]?.last_processed_at ?? null // Seconds since the last write for this network — a stalled ingester is // visible to `/status` polling without needing Prometheus. Null until the // network has ingested at least once. - ingestLagSeconds: lastProcessedAt + const ingestLagSeconds = lastProcessedAt ? Math.max(0, Math.round((Date.now() - new Date(lastProcessedAt).getTime()) / 1000)) - : null, + : null + + const staleAfterSeconds = config.status.staleAfterSeconds + const dataStatus = + ingestLagSeconds === null + ? 'empty' + : ingestLagSeconds > staleAfterSeconds + ? 'stale' + : 'fresh' + + const body = { + ok: true, + network, + watchedPairs: getNetworkConfig(network).pairs.map(p => p.pairKey), + lastIndexedLedger: result.rows[0]?.last_ledger ?? null, + lastProcessedAt, + ingestLagSeconds, + process: { + status: 'up' as const, + uptimeSeconds: Math.floor(process.uptime()), + }, + data: { + status: dataStatus, + ingestLagSeconds, + staleAfterSeconds, + }, + } + + if (dataStatus === 'stale') { + // Stale data is the paging condition: the API still answers (process is + // up), but the prices behind it are older than the staleness budget. + reply.header('Retry-After', '60') + return reply.status(503).send(body) + } + + return body } - }) + ) // GET /price/:assetA/:assetB diff --git a/src/api/schemas.ts b/src/api/schemas.ts index c7db8496..e4609396 100644 --- a/src/api/schemas.ts +++ b/src/api/schemas.ts @@ -16,12 +16,35 @@ import Ajv from 'ajv' import type { FastifyInstance, FastifySchema } from 'fastify' -/** GET /status */ +/** + * GET /status + * + * Two independent health verdicts, so an up process and stale data never look + * alike: + * - `process` describes this HTTP process (it is answering, so `up`). + * - `data` describes the ingester behind the selected network, derived from + * the age of its newest write. + * + * The flat `lastIndexedLedger` / `lastProcessedAt` / `ingestLagSeconds` fields + * predate the split and are kept as mirrors of the `data` values so existing + * consumers do not break. See docs/status.md for the contract. + */ export const statusResponseSchema = { type: 'object', - required: ['ok', 'network', 'watchedPairs', 'lastIndexedLedger', 'lastProcessedAt', 'ingestLagSeconds'], + required: [ + 'ok', + 'network', + 'watchedPairs', + 'lastIndexedLedger', + 'lastProcessedAt', + 'ingestLagSeconds', + 'process', + 'data', + ], additionalProperties: false, properties: { + // Process health only: true whenever this process can serve the request. + // It says nothing about whether the prices behind it are current. ok: { type: 'boolean' }, network: { type: 'string', enum: ['testnet', 'mainnet'] }, watchedPairs: { type: 'array', items: { type: 'string' } }, @@ -33,6 +56,25 @@ export const statusResponseSchema = { // Seconds since the last ingester write for this network, null before the // first one. Lets `/status` reveal a stalled ingester on its own. ingestLagSeconds: { type: ['integer', 'null'] }, + process: { + type: 'object', + required: ['status', 'uptimeSeconds'], + additionalProperties: false, + properties: { + status: { type: 'string', enum: ['up'] }, + uptimeSeconds: { type: 'integer' }, + }, + }, + data: { + type: 'object', + required: ['status', 'ingestLagSeconds', 'staleAfterSeconds'], + additionalProperties: false, + properties: { + status: { type: 'string', enum: ['fresh', 'stale', 'empty'] }, + ingestLagSeconds: { type: ['integer', 'null'] }, + staleAfterSeconds: { type: 'integer' }, + }, + }, }, } as const diff --git a/src/config.ts b/src/config.ts index 81b84827..63b124e9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -254,6 +254,11 @@ const globalConfig = { http: { timeoutMs: parseInt(process.env.HTTP_TIMEOUT_MS ?? '10000', 10), }, + status: { + // How old the newest indexer write may be before `/status` calls the data + // stale and answers 503. See docs/status.md. + staleAfterSeconds: parseInt(process.env.STATUS_STALE_AFTER_SECONDS ?? '300', 10), + }, } as const // ── Per-network map ─────────────────────────────────────────────────────────── @@ -334,4 +339,5 @@ export const config = { indexer: globalConfig.indexer, cache: globalConfig.cache, http: globalConfig.http, + status: globalConfig.status, } From 22ae9ff60a6ac7c4f5367ee3f36cab799aeba2bb Mon Sep 17 00:00:00 2001 From: Azintar Date: Sat, 24 Oct 2026 19:10:45 +0000 Subject: [PATCH 2/3] revert(status): drop the process/data health changes These duplicate work already on main through #533 (#273). This PR is for #275 only. --- .env.example | 4 - README.md | 21 ---- docs/status.md | 162 --------------------------- openapi.json | 207 +---------------------------------- openapi.yaml | 199 +-------------------------------- src/__tests__/status.test.ts | 60 ---------- src/api/rest.ts | 72 ------------ src/api/schemas.ts | 38 ------- src/config.ts | 5 - 9 files changed, 2 insertions(+), 766 deletions(-) delete mode 100644 docs/status.md diff --git a/.env.example b/.env.example index c460e733..bd073ffd 100644 --- a/.env.example +++ b/.env.example @@ -42,10 +42,6 @@ SNAPSHOT_CACHE_TTL=15 API_KEY_CACHE_TTL=30 # Default timeout for outbound HTTP requests in milliseconds (default: 10000) HTTP_TIMEOUT_MS=10000 -# How old the newest ingester write may be before GET /status reports -# data.status "stale" and answers 503. Keep it well above POLL_INTERVAL_MS. -# Default: 300 seconds. See docs/status.md. -STATUS_STALE_AFTER_SECONDS=300 # --- Outbound request policy (SSRF) --- # Comma-separated hostnames the app may reach when a caller supplies the target diff --git a/README.md b/README.md index a12da0ff..24302db6 100644 --- a/README.md +++ b/README.md @@ -119,11 +119,6 @@ npx @modelcontextprotocol/inspector npm run mcp:server | GET | `/movers` | Get biggest gainers, biggest losers, and volume movers | | GET | `/benchmark/:asset?target=USD` | Peg-deviation statistics versus a target | | GET | `/basket?asset=XLM&asset=USDC&weight=0.6&weight=0.4` | Weighted price of a basket of assets | -| GET | `/status` | Process and indexer health ([contract](docs/status.md)) | -| GET | `/status` | Indexer health | -| GET | `/snapshot?network=testnet\|mainnet` | Whole-network snapshot: venues, TVL, 24h volume, top pairs, peg readings and data freshness in one cached, timestamped object | -| GET | `/live` | Liveness probe — no dependency checks (see [`docs/health.md`](docs/health.md)) | -| GET | `/ready` | Readiness probe — boot complete + dependencies reachable | | GET | `/correlation` | Compute price correlation matrix across a basket of assets over a window | | GET | `/discovery/resources?type=&payTo=&network=&extensions=&limit=&offset=` | Bazaar catalog of x402-discoverable resources (spec: [`bazaar`](https://github.com/x402-foundation/x402/blob/main/specs/extensions/bazaar.md)) | | GET | `/discovery/search` | Natural language search with ranking and quality evaluation for discoverable resources | @@ -542,16 +537,6 @@ Lens gates `/price`, `/pools`, and `/candles` behind x402 micropayments on Stell ```bash curl http://localhost:3002/status -# { -# "ok": true, -# "network": "testnet", -# "watchedPairs": ["XLM:native/USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"], -# "lastIndexedLedger": 53842917, -# "lastProcessedAt": "2026-10-20T16:45:11.220Z", -# "ingestLagSeconds": 4, -# "process": { "status": "up", "uptimeSeconds": 86400 }, -# "data": { "status": "fresh", "ingestLagSeconds": 4, "staleAfterSeconds": 300 } -# } ``` ```json @@ -603,12 +588,6 @@ probe: condition: "body.health.data == 'stale' || body.data.ingestLagSeconds > 300" ``` -Process health and data health are separate: `process.status` is `up` whenever -the process answers, while `data.status` is `fresh`, `stale` or `empty`. A -stale ingester answers **`503`** (the only paging condition); `empty` stays -`200`. See [`docs/status.md`](docs/status.md) for the full shape, status-code -contract and monitor configuration examples. - ### 2. Paid request without `X-PAYMENT` → `402` with payment requirements ```bash diff --git a/docs/status.md b/docs/status.md deleted file mode 100644 index 9dd5a091..00000000 --- a/docs/status.md +++ /dev/null @@ -1,162 +0,0 @@ -# `GET /status` — health contract - -`/status` is the endpoint uptime monitors and integrators hit first. It is -public, needs no API key and is not gated by x402. - -It reports **two independent health verdicts**: - -| Field | Answers | Values | -|---|---|---| -| `process.status` | Is this HTTP process able to answer? | `up` | -| `data.status` | Is the ingester behind the selected network current? | `fresh`, `stale`, `empty` | - -The process can be up while every venue is stale. Those two states must not -look alike, so they are never collapsed into one flag: `ok` and `process` are -about the process, `data` is about the data. - -## Response shape - -```json -{ - "ok": true, - "network": "mainnet", - "watchedPairs": [ - "USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM" - ], - "lastIndexedLedger": 53842917, - "lastProcessedAt": "2026-10-20T16:45:11.220Z", - "ingestLagSeconds": 4, - "process": { - "status": "up", - "uptimeSeconds": 86400 - }, - "data": { - "status": "fresh", - "ingestLagSeconds": 4, - "staleAfterSeconds": 300 - } -} -``` - -| Field | Type | Meaning | -|---|---|---| -| `ok` | boolean | Process health only. `true` whenever the process serves this response. It says nothing about data freshness — read `data.status` or the HTTP status code for that. | -| `network` | `"testnet"` \| `"mainnet"` | The network this report covers, resolved from `?network=` / `x-network` and defaulting to the instance's `STELLAR_NETWORK`. | -| `watchedPairs` | string[] | Canonical pair keys this network's ingester watches. | -| `lastIndexedLedger` | integer \| null | Newest ledger recorded for this network. `null` until its SDEX ingester records a trade. | -| `lastProcessedAt` | ISO 8601 \| null | Timestamp of the newest ingester write for this network. `null` before the first one. | -| `ingestLagSeconds` | integer \| null | Seconds since `lastProcessedAt`. `null` when the network has never ingested. Top-level mirror of `data.ingestLagSeconds`. | -| `process.status` | `"up"` | The process is answering. A monitor learns the process is *down* from a failed connection, not from a body field. | -| `process.uptimeSeconds` | integer | Seconds since this process started. | -| `data.status` | `"fresh"` \| `"stale"` \| `"empty"` | `fresh` when `ingestLagSeconds <= staleAfterSeconds`; `stale` when it is greater; `empty` when the network has never ingested. | -| `data.ingestLagSeconds` | integer \| null | Same value as the top-level field, grouped with the verdict that uses it. | -| `data.staleAfterSeconds` | integer | The staleness budget in force (`STATUS_STALE_AFTER_SECONDS`, default `300`). | - -`lastIndexedLedger`, `lastProcessedAt` and `ingestLagSeconds` predate the -process/data split and are kept at the top level so existing consumers keep -working; new integrations can read everything from `data`. - -### `empty` is not `stale` - -A network that has never ingested has no lag to measure. Reporting it as stale -would page on every fresh deployment, so it is `empty` and answered with `200`. -A monitor that must know an ingester started can alert on -`data.status == "empty"` after its own startup grace period. - -## HTTP status codes - -| Code | When | Page? | -|---|---|---| -| `200` | `data.status` is `fresh` or `empty` | No | -| `503` | `data.status` is `stale` — newest write is older than `staleAfterSeconds` | **Yes** | -| `500` | Indexer state could not be read, so no health report could be produced | Yes | -| `400` | Unrecognised `?network=` value (only `testnet` / `mainnet`) | No — fix the monitor | - -`/status` is exempt from the API rate limiter, so it never answers `429`. A -monitor polling every few seconds will not exhaust a quota. - -**The paging condition is `503`.** It is the only status this endpoint returns -for "up, but its data is too old": the process answered, so the request itself -succeeded, but every price behind it is older than the staleness budget. A -monitor configuration that pages on any non-2xx response and treats `400` as a -configuration problem is correct. - -`503` responses carry the full status body (including `data.ingestLagSeconds` -and `Retry-After: 60`), so an alert can say *how far* behind the ingester is -instead of only that it is behind. - -## Tuning the staleness budget - -```bash -# Seconds the newest ingester write may age before /status reports stale. -# Default: 300 (5 minutes). -STATUS_STALE_AFTER_SECONDS=300 -``` - -Keep this comfortably above the ingester's poll interval (`POLL_INTERVAL_MS`, -default `5000`). A value near the poll interval pages on a single missed poll. - -## Monitor configuration examples - -### UptimeRobot (free plan) - -1. **Add New Monitor** → **HTTP(s)**. -2. URL: `https:///status?network=mainnet` — one monitor per - network you index. -3. Interval: 5 minutes. -4. Under **Alert Contacts**, tick your contact. Leave the advanced settings at - their defaults: UptimeRobot marks the monitor down on any non-2xx response, - which is exactly the paging condition above (`503` for stale data, `500` for - an unreadable health report). - -If you would rather alert only when the data is stale, use **Keyword** -monitoring instead: the keyword `"status":"stale"` must **not** exist in the -response body (the response is minified JSON, so match it without spaces). - -### Shell / cron (curl + jq) - -```bash -#!/usr/bin/env bash -set -euo pipefail - -# -f turns 503 (stale) and 500 into a non-zero exit code. -curl -fsS --max-time 5 \ - 'https:///status?network=mainnet' | - jq -e '.process.status == "up"' > /dev/null -``` - -Exit codes: `0` fresh, `22` stale (HTTP 503), `28` timeout/unreachable. Wire -the non-zero exit into your existing alerting. - -For "ingester never started" as well: - -```bash -curl -fsS --max-time 5 'https:///status?network=mainnet' | - jq -e '.data.status == "fresh"' -``` - -### Prometheus `blackbox_exporter` - -```yaml -modules: - lens_status: - prober: http - timeout: 5s - http: - valid_status_codes: [200] # 503 = stale data, alerts - preferred_ip_protocol: ip4 - fail_if_body_matches_regexp: - - '"status":"stale"' -``` - -Alert on `probe_success == 0` for 2 consecutive scrapes (10 minutes at a -`scrape_interval` of 5m) to ignore a single missed poll. - -### Kubernetes - -- **Liveness probe:** process health only. Use `httpGet: /status` and rely on - the connection succeeding — do not fail liveness on data staleness, or a - stalled Horizon will restart a healthy process in a loop. -- **Readiness probe:** if your consumers must never see stale prices, point it - at `/status` as well; `503` then removes the pod from the load balancer until - the ingester catches up. diff --git a/openapi.json b/openapi.json index 723fb255..0c401f5e 100644 --- a/openapi.json +++ b/openapi.json @@ -258,8 +258,7 @@ "/status": { "get": { "operationId": "getStatus", - "summary": "Get process health and indexer (data) health", - "description": "Public, unauthenticated health endpoint for uptime monitors and integrators. See [docs/status.md](https://github.com/Divinity-Labs/Lens/blob/main/docs/status.md) for the monitor configuration examples.\n\nProcess health and data health are reported separately. A process can be up while every venue's data is stale, and the two must not look alike:\n* `process.status` is `up` whenever this process answers the request.\n* `data.status` classifies the newest ingester write for the selected network as `fresh` (within `staleAfterSeconds`), `stale` (older than it), or `empty` (that network has not ingested yet).\n\nOnly `503` is a paging condition: it means the process is up but the selected network's data is stale. `200` covers `fresh` and `empty`. `400` (invalid `network`) is a monitor misconfiguration and `500` (unexpected error) means no report could be produced; neither is a staleness signal. `/status` is exempt from the API rate limiter, so it never answers 429.", + "summary": "Get API status and indexer state", "security": [], "parameters": [ { @@ -268,62 +267,6 @@ ], "responses": { "200": { - "description": "The process is up and the selected network's data is `fresh` or `empty` (never ingested, so there is no lag to judge).", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/StatusResponse" - }, - "examples": { - "fresh": { - "summary": "Ingester is current", - "value": { - "ok": true, - "network": "mainnet", - "watchedPairs": [ - "USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM" - ], - "lastIndexedLedger": 53842917, - "lastProcessedAt": "2026-10-20T16:45:11.220Z", - "ingestLagSeconds": 4, - "process": { - "status": "up", - "uptimeSeconds": 86400 - }, - "data": { - "status": "fresh", - "ingestLagSeconds": 4, - "staleAfterSeconds": 300 - } - } - }, - "empty": { - "summary": "Network has never ingested", - "value": { - "ok": true, - "network": "testnet", - "watchedPairs": [], - "lastIndexedLedger": null, - "lastProcessedAt": null, - "ingestLagSeconds": null, - "process": { - "status": "up", - "uptimeSeconds": 12 - }, - "data": { - "status": "empty", - "ingestLagSeconds": null, - "staleAfterSeconds": 300 - } - } - } - } - } - } - }, - "400": { - "description": "Unrecognised `network` value. Not a health signal — the monitor is misconfigured.", - "description": "API status", "description": "API status and health diagnostics", "headers": { "Cache-Control": { @@ -334,12 +277,7 @@ "application/json": { "schema": { "type": "object", - "required": [ - "error" - ], "properties": { - "error": { - "type": "string" "ok": { "type": "boolean" }, @@ -428,49 +366,10 @@ "description": "Whether this deployment has an ADMIN_TOKEN configured, so an operator can tell a rejected token from a disabled admin surface. Reports only that a token is set, never any part of its value. Admin routes answer 503 admin_not_configured when this is false." } } - }, - "example": { - "error": "Invalid network \"pubnet\" — expected one of: testnet, mainnet" } } } }, - "500": { - "description": "Unexpected error while reading indexer state. The process could not produce its health report, so the monitor should page." - }, - "503": { - "description": "Paging condition: the process is up but the selected network's data is `stale` — its newest write is older than `data.staleAfterSeconds`. The body is the full status shape, so a monitor can report how far behind the ingester is.", - "headers": { - "Retry-After": { - "description": "Seconds to wait before polling again.", - "schema": { - "type": "integer", - "example": 60 - } - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/StatusResponse" - }, - "example": { - "ok": true, - "network": "mainnet", - "watchedPairs": [ - "USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM" - ], - "lastIndexedLedger": 53800000, - "lastProcessedAt": "2026-10-20T16:05:11.220Z", - "ingestLagSeconds": 2404, - "process": { - "status": "up", - "uptimeSeconds": 86400 - }, - "data": { - "status": "stale", - "ingestLagSeconds": 2404, - "staleAfterSeconds": 300 "429": { "$ref": "#/components/responses/TooManyRequests" }, @@ -6027,110 +5926,6 @@ } } }, - "StatusResponse": { - "type": "object", - "description": "Process health and data health for one network. `ok` and `process` describe this HTTP process; `data` describes the ingester behind the selected network. The flat `lastIndexedLedger`, `lastProcessedAt` and `ingestLagSeconds` fields mirror the `data` values for older clients.", - "required": [ - "ok", - "network", - "watchedPairs", - "lastIndexedLedger", - "lastProcessedAt", - "ingestLagSeconds", - "process", - "data" - ], - "additionalProperties": false, - "properties": { - "ok": { - "type": "boolean", - "description": "Process health only — true whenever this process can serve the request. It says nothing about whether the data is current; read `data.status` (or the HTTP status code) for that." - }, - "network": { - "type": "string", - "enum": [ - "testnet", - "mainnet" - ], - "description": "The network this report covers." - }, - "watchedPairs": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Canonical pair keys this network's ingester watches." - }, - "lastIndexedLedger": { - "type": "integer", - "nullable": true, - "description": "Newest ledger recorded for this network, or null before its SDEX ingester records a trade. Mirror of the data behind `data.status`." - }, - "lastProcessedAt": { - "type": "string", - "format": "date-time", - "nullable": true, - "description": "Timestamp of the newest ingester write for this network, or null before the first one." - }, - "ingestLagSeconds": { - "type": "integer", - "nullable": true, - "description": "Seconds since `lastProcessedAt`, or null before the first write. Kept as a top-level mirror of `data.ingestLagSeconds`." - }, - "process": { - "type": "object", - "description": "Health of the HTTP process itself.", - "required": [ - "status", - "uptimeSeconds" - ], - "additionalProperties": false, - "properties": { - "status": { - "type": "string", - "enum": [ - "up" - ], - "description": "`up` whenever the process answers. A non-200 response is only produced for the paging condition, never because the process has stopped answering (that is a connection failure)." - }, - "uptimeSeconds": { - "type": "integer", - "description": "Seconds since this process started." - } - } - }, - "data": { - "type": "object", - "description": "Freshness of the ingester data behind the selected network. This is independent of `process.status`: data can be stale while the process is up.", - "required": [ - "status", - "ingestLagSeconds", - "staleAfterSeconds" - ], - "additionalProperties": false, - "properties": { - "status": { - "type": "string", - "enum": [ - "fresh", - "stale", - "empty" - ], - "description": "`fresh` when `ingestLagSeconds` is at most `staleAfterSeconds`; `stale` when it is greater (the only paging condition, answered with 503); `empty` when the network has never ingested, so no lag exists yet (answered with 200)." - }, - "ingestLagSeconds": { - "type": "integer", - "nullable": true, - "description": "Seconds since the newest ingester write, or null when `status` is `empty`." - }, - "staleAfterSeconds": { - "type": "integer", - "description": "The configured staleness budget (`STATUS_STALE_AFTER_SECONDS`, default 300). A lag greater than this makes `status` `stale`." - } - } - } - } - }, "GraphQlResponse": { "type": "object", "additionalProperties": true, diff --git a/openapi.yaml b/openapi.yaml index 60e90e9f..72d5778b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -204,81 +204,12 @@ paths: /status: get: operationId: getStatus - summary: Get process health and indexer (data) health - description: >- - Public, unauthenticated health endpoint for uptime monitors and - integrators. See [docs/status.md](https://github.com/Divinity-Labs/Lens/blob/main/docs/status.md) - for the monitor configuration examples. - - - Process health and data health are reported separately. A process can - be up while every venue's data is stale, and the two must not look - alike: - - * `process.status` is `up` whenever this process answers the request. - - * `data.status` classifies the newest ingester write for the selected - network as `fresh` (within `staleAfterSeconds`), `stale` (older than - it), or `empty` (that network has not ingested yet). - - - Only `503` is a paging condition: it means the process is up but the - selected network's data is stale. `200` covers `fresh` and `empty`. - `400` (invalid `network`) is a monitor misconfiguration and `500` - (unexpected error) means no report could be produced; neither is a - staleness signal. `/status` is exempt from the API rate limiter, so it - never answers 429. + summary: Get API status and indexer state security: [] parameters: - $ref: '#/components/parameters/NetworkQuery' responses: '200': - description: >- - The process is up and the selected network's data is `fresh` or - `empty` (never ingested, so there is no lag to judge). - content: - application/json: - schema: - $ref: '#/components/schemas/StatusResponse' - examples: - fresh: - summary: Ingester is current - value: - ok: true - network: mainnet - watchedPairs: - - USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM - lastIndexedLedger: 53842917 - lastProcessedAt: '2026-10-20T16:45:11.220Z' - ingestLagSeconds: 4 - process: - status: up - uptimeSeconds: 86400 - data: - status: fresh - ingestLagSeconds: 4 - staleAfterSeconds: 300 - empty: - summary: Network has never ingested - value: - ok: true - network: testnet - watchedPairs: [] - lastIndexedLedger: null - lastProcessedAt: null - ingestLagSeconds: null - process: - status: up - uptimeSeconds: 12 - data: - status: empty - ingestLagSeconds: null - staleAfterSeconds: 300 - '400': - description: >- - Unrecognised `network` value. Not a health signal — the monitor is - misconfigured. - description: API status description: API status and health diagnostics headers: Cache-Control: @@ -287,47 +218,7 @@ paths: application/json: schema: type: object - required: [error] properties: - error: - type: string - example: - error: 'Invalid network "pubnet" — expected one of: testnet, mainnet' - '500': - description: >- - Unexpected error while reading indexer state. The process could not - produce its health report, so the monitor should page. - '503': - description: >- - Paging condition: the process is up but the selected network's - data is `stale` — its newest write is older than - `data.staleAfterSeconds`. The body is the full status shape, so a - monitor can report how far behind the ingester is. - headers: - Retry-After: - description: Seconds to wait before polling again. - schema: - type: integer - example: 60 - content: - application/json: - schema: - $ref: '#/components/schemas/StatusResponse' - example: - ok: true - network: mainnet - watchedPairs: - - USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN/XLM - lastIndexedLedger: 53800000 - lastProcessedAt: '2026-10-20T16:05:11.220Z' - ingestLagSeconds: 2404 - process: - status: up - uptimeSeconds: 86400 - data: - status: stale - ingestLagSeconds: 2404 - staleAfterSeconds: 300 ok: type: boolean status: @@ -3840,94 +3731,6 @@ components: items: type: string - StatusResponse: - type: object - description: >- - Process health and data health for one network. `ok` and `process` - describe this HTTP process; `data` describes the ingester behind the - selected network. The flat `lastIndexedLedger`, `lastProcessedAt` and - `ingestLagSeconds` fields mirror the `data` values for older clients. - required: - [ok, network, watchedPairs, lastIndexedLedger, lastProcessedAt, ingestLagSeconds, process, data] - additionalProperties: false - properties: - ok: - type: boolean - description: >- - Process health only — true whenever this process can serve the - request. It says nothing about whether the data is current; read - `data.status` (or the HTTP status code) for that. - network: - type: string - enum: [testnet, mainnet] - description: The network this report covers. - watchedPairs: - type: array - items: - type: string - description: Canonical pair keys this network's ingester watches. - lastIndexedLedger: - type: integer - nullable: true - description: >- - Newest ledger recorded for this network, or null before its SDEX - ingester records a trade. Mirror of the data behind `data.status`. - lastProcessedAt: - type: string - format: date-time - nullable: true - description: >- - Timestamp of the newest ingester write for this network, or null - before the first one. - ingestLagSeconds: - type: integer - nullable: true - description: >- - Seconds since `lastProcessedAt`, or null before the first write. - Kept as a top-level mirror of `data.ingestLagSeconds`. - process: - type: object - description: Health of the HTTP process itself. - required: [status, uptimeSeconds] - additionalProperties: false - properties: - status: - type: string - enum: [up] - description: >- - `up` whenever the process answers. A non-200 response is only - produced for the paging condition, never because the process - has stopped answering (that is a connection failure). - uptimeSeconds: - type: integer - description: Seconds since this process started. - data: - type: object - description: >- - Freshness of the ingester data behind the selected network. This is - independent of `process.status`: data can be stale while the - process is up. - required: [status, ingestLagSeconds, staleAfterSeconds] - additionalProperties: false - properties: - status: - type: string - enum: [fresh, stale, empty] - description: >- - `fresh` when `ingestLagSeconds` is at most `staleAfterSeconds`; - `stale` when it is greater (the only paging condition, answered - with 503); `empty` when the network has never ingested, so no - lag exists yet (answered with 200). - ingestLagSeconds: - type: integer - nullable: true - description: Seconds since the newest ingester write, or null when `status` is `empty`. - staleAfterSeconds: - type: integer - description: >- - The configured staleness budget (`STATUS_STALE_AFTER_SECONDS`, - default 300). A lag greater than this makes `status` `stale`. - GraphQlResponse: type: object additionalProperties: true diff --git a/src/__tests__/status.test.ts b/src/__tests__/status.test.ts index e2f76d0d..9c4995e2 100644 --- a/src/__tests__/status.test.ts +++ b/src/__tests__/status.test.ts @@ -43,7 +43,6 @@ vi.mock('../config', () => ({ config: { pairs: pairsByNetwork.testnet, cache: { priceTtl: 10 }, - status: { staleAfterSeconds: 300 }, }, activeNetwork: 'testnet', getNetworkConfig: (network: string) => ({ @@ -120,61 +119,6 @@ describe('GET /status', () => { expect(body.ingestLagSeconds).toBeLessThan(60) }) - it('reports process health and data health as separate fields', async () => { - const app = await buildApp() - - const res = await app.inject({ method: 'GET', url: '/status?network=mainnet' }) - const body = res.json() - - expect(res.statusCode).toBe(200) - expect(body.ok).toBe(true) - expect(body.process).toEqual({ - status: 'up', - uptimeSeconds: expect.any(Number), - }) - expect(Number.isInteger(body.process.uptimeSeconds)).toBe(true) - expect(body.data).toEqual({ - status: 'fresh', - ingestLagSeconds: body.ingestLagSeconds, - staleAfterSeconds: 300, - }) - }) - - it('pages with 503 only when data is stale, while still reporting the process as up', async () => { - mockQuery.mockResolvedValue({ - rows: [{ last_ledger: 222222, last_processed_at: new Date(Date.now() - 3_600_000) }], - }) - const app = await buildApp() - - const res = await app.inject({ method: 'GET', url: '/status?network=mainnet' }) - const body = res.json() - - expect(res.statusCode).toBe(503) - expect(res.headers['retry-after']).toBe('60') - // The process is healthy; only its data is not. - expect(body.ok).toBe(true) - expect(body.process.status).toBe('up') - expect(body.data.status).toBe('stale') - expect(body.data.ingestLagSeconds).toBeGreaterThan(300) - expect(body.data.staleAfterSeconds).toBe(300) - expect(body.lastIndexedLedger).toBe(222222) - expect(body.lastProcessedAt).not.toBeNull() - }) - - it('treats a lag of exactly staleAfterSeconds as fresh', async () => { - mockQuery.mockResolvedValue({ - rows: [{ last_ledger: 222222, last_processed_at: new Date(Date.now() - 300_000) }], - }) - const app = await buildApp() - - const res = await app.inject({ method: 'GET', url: '/status?network=mainnet' }) - - // Rounded lag can land on 299/300; either way it must not be stale. - expect(res.statusCode).toBe(200) - expect(res.json().data.status).toBe('fresh') - }) - - it('returns nulls — never another network\'s data — when a network has not ingested', async () => { it('returns nulls — never another network \'s data — when a network has not ingested', async () => { mockQuery.mockResolvedValue({ rows: [] }) const app = await buildApp() @@ -182,14 +126,10 @@ describe('GET /status', () => { const res = await app.inject({ method: 'GET', url: '/status?network=mainnet' }) const body = res.json() - expect(res.statusCode).toBe(200) expect(body.network).toBe('mainnet') expect(body.lastIndexedLedger).toBeNull() expect(body.lastProcessedAt).toBeNull() expect(body.ingestLagSeconds).toBeNull() - // A network that has never ingested is not stale — there is no lag to - // measure — so it stays a 200 and is distinguishable via data.status. - expect(body.data.status).toBe('empty') }) it('rejects an unrecognised network before answering', async () => { diff --git a/src/api/rest.ts b/src/api/rest.ts index 35fd4834..edc76157 100644 --- a/src/api/rest.ts +++ b/src/api/rest.ts @@ -77,12 +77,6 @@ export async function registerRESTRoutes(app: FastifyInstance) { // up the validating serializer. installResponseValidation(app) - // GET /status — public health/monitoring endpoint (no API key required). - // - // Process health and data health are reported separately: this process can - // be up while every venue's data is stale, and a monitor must be able to tell - // those apart. The only non-2xx outcome that means "page someone" is 503, - // returned when the selected network's data is stale (see docs/status.md). /** * How long the last ingester write may be stale before `/status` says so. * @@ -98,72 +92,6 @@ export async function registerRESTRoutes(app: FastifyInstance) { '/status', { config: { public: true }, - schema: { response: { 200: statusResponseSchema, 503: statusResponseSchema } }, - }, - async (req, reply) => { - const network = req.network ?? activeNetwork - const result = await pgPool.query( - `SELECT last_ledger, last_processed_at - FROM indexer_state - WHERE network = $1 - ORDER BY updated_at DESC - LIMIT 1`, - [network] - ) - const lastProcessedAt = result.rows[0]?.last_processed_at ?? null - // Seconds since the last write for this network — a stalled ingester is - // visible to `/status` polling without needing Prometheus. Null until the - // network has ingested at least once. - const ingestLagSeconds = lastProcessedAt - ? Math.max(0, Math.round((Date.now() - new Date(lastProcessedAt).getTime()) / 1000)) - : null - - const staleAfterSeconds = config.status.staleAfterSeconds - const dataStatus = - ingestLagSeconds === null - ? 'empty' - : ingestLagSeconds > staleAfterSeconds - ? 'stale' - : 'fresh' - - const body = { - ok: true, - network, - watchedPairs: getNetworkConfig(network).pairs.map(p => p.pairKey), - lastIndexedLedger: result.rows[0]?.last_ledger ?? null, - lastProcessedAt, - ingestLagSeconds, - process: { - status: 'up' as const, - uptimeSeconds: Math.floor(process.uptime()), - }, - data: { - status: dataStatus, - ingestLagSeconds, - staleAfterSeconds, - }, - } - - if (dataStatus === 'stale') { - // Stale data is the paging condition: the API still answers (process is - // up), but the prices behind it are older than the staleness budget. - reply.header('Retry-After', '60') - return reply.status(503).send(body) - } - - return body - : null, - // Ingesters that have crashed at least once on this network, newest - // first. An ingester that crash-loops but restarts successfully is - // otherwise invisible here: the process is up and the port answers. - // Empty when nothing has crashed. Messages only — never a stack, since - // this is served over HTTP. - crashes: listCrashes(network).map(c => ({ - name: c.name, - network: c.network, - message: c.message, - at: c.at.toISOString(), - })), schema: { response: { 200: statusResponseSchema, diff --git a/src/api/schemas.ts b/src/api/schemas.ts index 49d87e16..c7a4ebe6 100644 --- a/src/api/schemas.ts +++ b/src/api/schemas.ts @@ -17,18 +17,6 @@ import Ajv from 'ajv' import type { FastifyInstance, FastifySchema } from 'fastify' /** - * GET /status - * - * Two independent health verdicts, so an up process and stale data never look - * alike: - * - `process` describes this HTTP process (it is answering, so `up`). - * - `data` describes the ingester behind the selected network, derived from - * the age of its newest write. - * - * The flat `lastIndexedLedger` / `lastProcessedAt` / `ingestLagSeconds` fields - * predate the split and are kept as mirrors of the `data` values so existing - * consumers do not break. See docs/status.md for the contract. - */ * One dependency probe result, shared by `/ready`. * * `reason` is a fixed phrase rather than the raw error: connection errors carry @@ -82,7 +70,6 @@ export const statusResponseSchema = { type: 'object', required: [ 'ok', - 'network', 'status', 'network', 'health', @@ -92,13 +79,9 @@ export const statusResponseSchema = { 'lastIndexedLedger', 'lastProcessedAt', 'ingestLagSeconds', - 'process', - 'data', ], additionalProperties: false, properties: { - // Process health only: true whenever this process can serve the request. - // It says nothing about whether the prices behind it are current. ok: { type: 'boolean' }, status: { type: 'string', enum: ['healthy', 'degraded', 'unhealthy'] }, network: { type: 'string', enum: ['testnet', 'mainnet'] }, @@ -133,27 +116,6 @@ export const statusResponseSchema = { lastIndexedLedger: { type: ['integer', 'null'] }, lastProcessedAt: {}, ingestLagSeconds: { type: ['integer', 'null'] }, - process: { - type: 'object', - required: ['status', 'uptimeSeconds'], - additionalProperties: false, - properties: { - status: { type: 'string', enum: ['up'] }, - uptimeSeconds: { type: 'integer' }, - }, - }, - data: { - type: 'object', - required: ['status', 'ingestLagSeconds', 'staleAfterSeconds'], - additionalProperties: false, - properties: { - status: { type: 'string', enum: ['fresh', 'stale', 'empty'] }, - ingestLagSeconds: { type: ['integer', 'null'] }, - staleAfterSeconds: { type: 'integer' }, - // Ingesters that have crashed at least once on this network, newest first, - // and empty when none have. Present so a crash-looping ingester that keeps - // restarting is visible here rather than only in the logs. `message` only - // — never a stack, since this is served over HTTP. adminConfigured: { type: 'boolean' }, crashes: { type: 'array', diff --git a/src/config.ts b/src/config.ts index 2f5651d3..f0fe496a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -585,10 +585,6 @@ const globalConfig = { http: { timeoutMs: parseInt(process.env.HTTP_TIMEOUT_MS ?? '10000', 10), }, - status: { - // How old the newest indexer write may be before `/status` calls the data - // stale and answers 503. See docs/status.md. - staleAfterSeconds: parseInt(process.env.STATUS_STALE_AFTER_SECONDS ?? '300', 10), ws: { pingIntervalMs: parseInt(process.env.WS_PING_INTERVAL_MS ?? '30000', 10), maxMissedPings: parseInt(process.env.WS_MAX_MISSED_PINGS ?? '2', 10), @@ -702,7 +698,6 @@ export const config = { cache: globalConfig.cache, plausibility: globalConfig.plausibility, http: globalConfig.http, - status: globalConfig.status, ws: globalConfig.ws, outbound: globalConfig.outbound, mcp: globalConfig.mcp, From 0e1959b8d28926c89ab777a5ce5f494e3fcf0e78 Mon Sep 17 00:00:00 2001 From: Azintar Date: Sat, 24 Oct 2026 19:16:36 +0000 Subject: [PATCH 3/3] docs(readme): generate the endpoint table as part of openapi:gen and fail CI on drift (#275) --- .../readme-endpoint-table-in-openapi-gen.md | 5 ++++ .github/workflows/ci.yml | 2 ++ README.md | 4 +++ package.json | 3 +- scripts/generate-readme.ts | 28 +++++++++++++++++-- tests/readme.test.ts | 10 +++++++ 6 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 .changeset/readme-endpoint-table-in-openapi-gen.md diff --git a/.changeset/readme-endpoint-table-in-openapi-gen.md b/.changeset/readme-endpoint-table-in-openapi-gen.md new file mode 100644 index 00000000..0d889819 --- /dev/null +++ b/.changeset/readme-endpoint-table-in-openapi-gen.md @@ -0,0 +1,5 @@ +--- +'lens': patch +--- + +`npm run openapi:gen` now also regenerates the README endpoint table from `openapi.yaml`, and CI runs `npm run readme:check`, which fails when the committed README differs from a fresh generation (#275). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2929c8a9..cf7ebfd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: run: git diff --exit-code -- clients/typescript - name: Typecheck the generated TypeScript client run: npm run client:typecheck + - name: Fail if the committed README tables are out of date with openapi.yaml + run: npm run readme:check - run: npx tsc --noEmit - run: npm run build - run: npm test --if-present diff --git a/README.md b/README.md index 24302db6..ca29daae 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,10 @@ npx @modelcontextprotocol/inspector npm run mcp:server | GET/POST | `/graphql` | GraphQL query/mutation endpoint (GraphiQL IDE at `/graphiql`, docs: [`docs/graphql.md`](docs/graphql.md)) | +The table above is generated from [`openapi.yaml`](openapi.yaml) by `npm run openapi:gen`; +edit the spec rather than the table. CI runs `npm run readme:check` and fails when +the committed table differs from a fresh generation. + Every non-internal route above has a matching entry in [`openapi.yaml`](openapi.yaml); `tests/openapi.test.ts` boots the server and fails if a registered route is missing from the spec. Operator-only and non-HTTP routes (`/admin/keys*`, diff --git a/package.json b/package.json index b4899fe7..5e083709 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,9 @@ "test": "vitest run", "smoke": "node scripts/deployed-smoke.mjs", "test:smoke": "node --test scripts/test-deployed-smoke.mjs", - "openapi:gen": "tsx scripts/generate-openapi.ts", + "openapi:gen": "tsx scripts/generate-openapi.ts && tsx scripts/generate-readme.ts", "readme:gen": "tsx scripts/generate-readme.ts", + "readme:check": "tsx scripts/generate-readme.ts --check", "spec:validate-live": "tsx scripts/validate-live-spec.ts", "client:gen": "openapi-typescript openapi.yaml -o clients/typescript/src/schema.d.ts", "client:typecheck": "tsc -p clients/typescript/tsconfig.json --noEmit", diff --git a/scripts/generate-readme.ts b/scripts/generate-readme.ts index e5a9fde5..f8960cf1 100644 --- a/scripts/generate-readme.ts +++ b/scripts/generate-readme.ts @@ -277,8 +277,30 @@ export function generateReadme(): void { writeFileSync(readmePath, renderReadme()) } -// CLI entry point +/** + * Compares the committed README.md with a fresh render and reports whether it + * has drifted. Line endings are normalised first, as in `tests/readme.test.ts`, + * so a CRLF checkout of an identical table is not reported as drift. + */ +export function readmeIsCurrent(content?: string): boolean { + const committed = (content ?? readFileSync(readmePath, 'utf8')).replace(/\r\n/g, '\n') + return renderReadme(committed) === committed +} + +// CLI entry point. `--check` (`npm run readme:check`, run in CI) writes +// nothing and exits non-zero when README.md differs from a fresh generation. if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { - generateReadme() - console.log('Successfully generated README.md venues and endpoints sections') + if (process.argv.includes('--check')) { + if (!readmeIsCurrent()) { + console.error( + 'README.md is out of date: its generated tables (endpoints from openapi.yaml, venues from src/ingest/venues) differ from a fresh generation.\n' + + 'Run `npm run openapi:gen` and commit the result.', + ) + process.exit(1) + } + console.log('README.md venue and endpoint tables are up to date') + } else { + generateReadme() + console.log('Successfully generated README.md venues and endpoints sections') + } } diff --git a/tests/readme.test.ts b/tests/readme.test.ts index 0b0c28e0..c74a54e9 100644 --- a/tests/readme.test.ts +++ b/tests/readme.test.ts @@ -9,6 +9,7 @@ import { renderVenuesTable, renderEndpointsTable, renderReadme, + readmeIsCurrent, VENUES_START_TAG, VENUES_END_TAG, ENDPOINTS_START_TAG, @@ -125,6 +126,15 @@ describe('README venue list and endpoint table generators', () => { expect(currentEndpoints).not.toContain('/fake/nonexistent/endpoint') }) + it('readme:check passes on the committed README and fails once the endpoint table is edited by hand', () => { + const committed = readFileSync(readmePath, 'utf8') + expect(readmeIsCurrent(committed)).toBe(true) + + const edited = committed.replace('| GET | `/tvl` |', '| GET | `/tvl-renamed` |') + expect(edited).not.toBe(committed) + expect(readmeIsCurrent(edited)).toBe(false) + }) + it('renders README deterministically in-place', () => { const initial = readFileSync(readmePath, 'utf8') const rendered = renderReadme(initial)