feat(backend): SEP-0001 retry with backoff; Payment Processor Telemetry sanitization, health and retries

backend/SEP0001_GENERATOR_GUIDE.md

@@ -316,6 +316,33 @@ console.log('Accounts:', toml.ACCOUNTS);
 - Wait 1 hour for changes to propagate
 - Clear CDN cache if using a CDN
 
+## Retry with Exponential Backoff (#1459)
+
+The only I/O behind `GET /.well-known/stellar.toml` is the merchant lookup.
+It is retried when it fails transiently (`src/lib/sep0001-retry.js`):
+
+- Retried: network errors, 408/429/5xx, connection-class Postgres errors.
+- Not retried: unknown or invalid merchant ids, PostgREST client errors.
+- Delay: full jitter, `random(0, min(max, base * 2^n))`.
+
+| Variable | Default | Clamp |
+|---|---|---|
+| `SEP0001_RETRY_MAX_ATTEMPTS` | 3 | 1-5 |
+| `SEP0001_RETRY_BASE_DELAY_MS` | 100 | 0-5000 |
+| `SEP0001_RETRY_MAX_DELAY_MS` | 1000 | base-5000 |
+
+If the lookup is still failing transiently after the last attempt, the
+endpoint answers **503** with `Retry-After: 5` instead of 500, so wallets
+retry later instead of treating the domain as having no stellar.toml.
+Deterministic lookup errors still return 500. Every error response carries
+`Cache-Control: no-store`, so a CDN never caches a failure for the hour that
+successful responses are cached.
+
+Retry outcomes are exported as
+`sep0001_merchant_lookup_retries_total{result="scheduled|recovered|exhausted"}`
+on `GET /metrics`. Logs carry the error code, attempt count and message, never
+the merchant record.
+
 ## Security Considerations
 
 1. **Public Information**: stellar.toml is publicly accessible

backend/docs/PAYMENT_PROCESSOR.md

@@ -4,7 +4,9 @@ Architecture, granular metrics, and testing guide for the Payment Processor
 module (`src/routes/payments.js` + `src/services/paymentService.js`).
 
 Covers issues **#1086** (end-to-end testing), **#1087** (legacy refactor),
-**#1088** (granular metrics) and **#1089** (load testing).
+**#1088** (granular metrics), **#1089** (load testing), and Payment Processor
+Telemetry: **#1477** (payload sanitization and strict validation), **#1478**
+(alert metrics and health) and **#1479** (retry with exponential backoff).
 
 ---
 

backend/docs/alerts/payment-processor.rules.yml

@@ -0,0 +1,82 @@
+# Prometheus alerting rules for Payment Processor Telemetry (issues #1477-#1479).
+#
+# Load with:   rule_files: ["docs/alerts/payment-processor.rules.yml"]
+# Validate:    promtool check rules docs/alerts/payment-processor.rules.yml
+#
+# Metric reference: docs/PAYMENT_PROCESSOR.md (section 6)
+
+groups:
+  - name: payment-processor
+    rules:
+      - alert: PaymentProcessorUnhealthy
+        expr: max(payment_processor_health_state) >= 2
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Payment processor reports unhealthy
+          description: >
+            Session persistence, verification or refund errors are above the
+            unhealthy ratio over the rolling window. See
+            GET /health/payment-processor for the counts and reasons.
+
+      - alert: PaymentProcessorDegraded
+        expr: max(payment_processor_health_state) == 1
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: Payment processor degraded for 15 minutes
+          description: >
+            An error ratio is above the degraded threshold, telemetry payloads
+            are being rejected, or read retries are being exhausted.
+
+      - alert: PaymentProcessorSessionPersistenceFailures
+        expr: |
+          sum(rate(payment_processor_sessions_total{outcome="persistence_failed"}[10m]))
+            /
+          clamp_min(sum(rate(payment_processor_sessions_total{outcome=~"created|persistence_failed"}[10m])), 1e-9)
+            > 0.05
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: More than 5% of payment sessions fail to persist
+          description: >
+            Session inserts are failing after their own retries. Check
+            database health and the "Payment session operation failed" logs.
+
+      - alert: PaymentProcessorReadRetriesExhausted
+        expr: sum(increase(payment_processor_retries_total{result="exhausted"}[10m])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: Payment processor reads failed after all retries
+          description: >
+            Status or verification lookups kept failing with transient errors
+            until PAYMENT_PROCESSOR_RETRY_MAX_ATTEMPTS ran out. Customers saw
+            5xx on payment-status or verify-payment.
+
+      - alert: PaymentProcessorTelemetryRejections
+        expr: sum(increase(payment_processor_telemetry_rejected_total[15m])) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: Payment processor telemetry payloads are being rejected
+          description: >
+            A code path is recording an outcome, stage or duration outside the
+            allow-list. The business series are not affected (rejected
+            payloads are dropped) but the gap hides real traffic. Check the
+            "Payment processor telemetry payload rejected" log for the metric.
+
+      - alert: PaymentProcessorAssetLabelOverflow
+        expr: sum(increase(payment_processor_sessions_total{asset=~"other|invalid"}[1h])) > 100
+        labels:
+          severity: info
+        annotations:
+          summary: Many sessions recorded under asset "other" or "invalid"
+          description: >
+            Either the distinct asset label cap
+            (PAYMENT_PROCESSOR_TELEMETRY_MAX_ASSET_LABELS) is too low for real
+            traffic, or clients are sending malformed asset codes.

backend/src/lib/backoff-retry.js

@@ -0,0 +1,189 @@
+/**
+ * Shared full-jitter exponential backoff for idempotent reads (issues #1459,
+ * #1479).
+ *
+ * Used by the SEP-0001 Stellar Info Generator (merchant lookup) and the
+ * Payment Processor (status / verification lookups). Both retry ONLY
+ * transient failures, classified by the same rules as the payment session
+ * writer (`isRetryableSessionError`): network errors, 408/429/5xx and
+ * connection-class Postgres errors. Validation, auth, not-found and
+ * constraint errors are never retried, so a retry can never turn a rejected
+ * request into an accepted one.
+ *
+ *   delay(n) = random(0, min(maxDelayMs, baseDelayMs * 2^n))
+ *
+ * Attempts and delays are clamped to per-caller ceilings so env/config can
+ * never make an HTTP request hang.
+ */
+
+import { logger as defaultLogger } from "./logger.js";
+import { computeBackoffDelay, isRetryableSessionError } from "./payment-session-retry.js";
+
+function clampInt(value, fallback, min, max) {
+  if (value === undefined || value === null || value === "") return fallback;
+  const n = Number(value);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(Math.max(Math.trunc(n), min), max);
+}
+
+/**
+ * Resolve retry options from explicit options, then `<envPrefix>_MAX_ATTEMPTS`
+ * / `_BASE_DELAY_MS` / `_MAX_DELAY_MS`, then defaults, clamped to ceilings.
+ *
+ * @param {object} options
+ * @param {Record<string,string|undefined>} env
+ * @param {{envPrefix:string, defaults:{maxAttempts:number, baseDelayMs:number, maxDelayMs:number}, maxAttemptsCeiling:number, maxDelayCeilingMs:number}} spec
+ */
+export function resolveBackoffOptions(options = {}, env = process.env, spec) {
+  const { envPrefix, defaults, maxAttemptsCeiling, maxDelayCeilingMs } = spec;
+  const maxAttempts = clampInt(
+    options.maxAttempts ?? env[`${envPrefix}_MAX_ATTEMPTS`],
+    defaults.maxAttempts,
+    1,
+    maxAttemptsCeiling,
+  );
+  const baseDelayMs = clampInt(
+    options.baseDelayMs ?? env[`${envPrefix}_BASE_DELAY_MS`],
+    defaults.baseDelayMs,
+    0,
+    maxDelayCeilingMs,
+  );
+  const maxDelayMs = clampInt(
+    options.maxDelayMs ?? env[`${envPrefix}_MAX_DELAY_MS`],
+    defaults.maxDelayMs,
+    baseDelayMs,
+    maxDelayCeilingMs,
+  );
+  return { maxAttempts, baseDelayMs, maxDelayMs };
+}
+
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Run `operation`, retrying transient failures with full-jitter backoff.
+ * The final error is rethrown annotated with `retryAttempts`.
+ *
+ * @template T
+ * @param {(ctx:{attempt:number}) => Promise<T>} operation
+ * @param {object} options
+ * @param {{maxAttempts:number, baseDelayMs:number, maxDelayMs:number}} options.resolved
+ * @param {string} [options.label]  Fixed, server-side operation name
+ * @param {(err:unknown) => boolean} [options.shouldRetry]
+ * @param {(ms:number) => Promise<void>} [options.sleep]
+ * @param {() => number} [options.random]
+ * @param {(info:{attempt:number, delayMs:number, error:unknown}) => void} [options.onRetry]
+ * @param {(info:{attempts:number}) => void} [options.onRecovered]
+ * @param {(info:{attempts:number, error:unknown}) => void} [options.onExhausted]
+ * @param {object} [options.context]  Extra, non-sensitive log fields
+ * @param {object} [options.logger]
+ * @returns {Promise<T>}
+ */
+export async function retryWithBackoff(operation, options) {
+  const {
+    resolved,
+    label = "operation",
+    shouldRetry = isRetryableSessionError,
+    sleep = defaultSleep,
+    random = Math.random,
+    onRetry,
+    onRecovered,
+    onExhausted,
+    context = {},
+    logger = defaultLogger,
+  } = options;
+
+  let lastError;
+  for (let attempt = 1; attempt <= resolved.maxAttempts; attempt += 1) {
+    try {
+      const result = await operation({ attempt });
+      if (attempt > 1) onRecovered?.({ attempts: attempt });
+      return result;
+    } catch (err) {
+      lastError = err;
+      const retryable = shouldRetry(err);
+      const exhausted = attempt >= resolved.maxAttempts;
+
+      if (!retryable || exhausted) {
+        if (err && typeof err === "object") err.retryAttempts = attempt;
+        if (retryable && exhausted) {
+          onExhausted?.({ attempts: attempt, error: err });
+          logger.error(
+            { ...context, label, attempts: attempt, err: err?.message, code: err?.code },
+            "Operation failed after exhausting retries",
+          );
+        }
+        throw err;
+      }
+
+      const delayMs = computeBackoffDelay(attempt - 1, resolved, random);
+      logger.warn(
+        {
+          ...context,
+          label,
+          attempt,
+          maxAttempts: resolved.maxAttempts,
+          delayMs,
+          err: err?.message,
+          code: err?.code,
+        },
+        "Operation failed, retrying with backoff",
+      );
+      onRetry?.({ attempt, delayMs, error: err });
+      await sleep(delayMs);
+    }
+  }
+
+  throw lastError;
+}
+
+/** Internal carrier so a Supabase `{ error }` result can drive the retry loop. */
+class RetryableResultError extends Error {
+  constructor(result) {
+    super(result?.error?.message ?? "retryable result error");
+    this.name = "RetryableResultError";
+    this.result = result;
+    this.code = result?.error?.code;
+  }
+}
+
+/**
+ * Retry a Supabase-style read that reports failures as `{ data, error }`
+ * instead of throwing. `runQuery` must build a NEW query on every call
+ * (PostgREST builders are single-use).
+ *
+ * Transient `error`s are retried; when retries run out, or the error is not
+ * transient, the last `{ data, error }` result is returned unchanged so the
+ * caller's existing error handling and status codes still apply. Thrown
+ * transient errors are retried and rethrown when exhausted.
+ *
+ * @template R
+ * @param {() => Promise<R & {error?: unknown}>} runQuery
+ * @param {Parameters<typeof retryWithBackoff>[1]} options
+ * @returns {Promise<R>}
+ */
+export async function retrySupabaseRead(runQuery, options) {
+  const shouldRetry = options.shouldRetry ?? isRetryableSessionError;
+  try {
+    return await retryWithBackoff(
+      async (ctx) => {
+        const result = await runQuery(ctx);
+        if (result?.error && shouldRetry(result.error)) {
+          throw new RetryableResultError(result);
+        }
+        return result;
+      },
+      {
+        ...options,
+        shouldRetry: (err) => err instanceof RetryableResultError || shouldRetry(err),
+      },
+    );
+  } catch (err) {
+    if (err instanceof RetryableResultError) {
+      if (err.result?.error && typeof err.result.error === "object") {
+        err.result.error.retryAttempts = err.retryAttempts;
+      }
+      return err.result;
+    }
+    throw err;
+  }
+}