feat: crypto payment resilience (#1683)

src/services/paymentMethodService.ts

@@ -1,6 +1,8 @@
 import { ethers } from 'ethers';
 
-import { getEvmRpcUrl } from '../config/evm';
+import { getEvmRpcUrls } from '../config/evm';
+import { getOrCreateResilientProvider } from './rpcProvider';
+import { exceedsLimit } from '../utils/decimalAmount';
 import { CHAIN_IDS } from '../utils/constants/values';
 import {
   PaymentMethod,
@@ -195,7 +197,7 @@ export class PaymentMethodService {
     }
 
     try {
-      const provider = new ethers.providers.JsonRpcProvider(getEvmRpcUrl(method.chainId));
+      const provider = getOrCreateResilientProvider(method.chainId, getEvmRpcUrls(method.chainId));
       const erc20Abi = [
         'function decimals() view returns (uint8)',
         'function symbol() view returns (string)',
@@ -307,29 +309,35 @@ export class PaymentMethodService {
     chainId: number
   ): Promise<{ sufficient: boolean; balance: string; symbol: string }> {
     try {
-      const provider = new ethers.providers.JsonRpcProvider(getEvmRpcUrl(chainId));
+      const provider = getOrCreateResilientProvider(chainId, getEvmRpcUrls(chainId));
       const conn = this.walletManager?.getConnection();
       if (!conn) {
         return { sufficient: false, balance: '0', symbol: method.tokenType };
       }
 
       let balance: ethers.BigNumber;
+      // Read the token's own decimals instead of assuming "USDC = 6,
+      // everything else = 18". Assuming wrong under- or over-reports the
+      // balance, which triggers spurious fallbacks or lets a charge through
+      // that cannot settle.
+      let decimals = 18;
 
       if (method.tokenType === TokenType.NATIVE) {
         balance = await provider.getBalance(conn.address);
       } else {
-        const erc20Abi = ['function balanceOf(address) view returns (uint256)'];
+        const erc20Abi = [
+          'function decimals() view returns (uint8)',
+          'function balanceOf(address) view returns (uint256)',
+        ];
         const contract = new ethers.Contract(method.tokenAddress, erc20Abi, provider);
+        decimals = await contract.decimals();
         balance = await contract.balanceOf(conn.address);
       }
 
-      const required = ethers.utils.parseUnits(
-        requiredAmount,
-        method.tokenType === TokenType.USDC ? 6 : 18
-      );
+      const required = ethers.utils.parseUnits(requiredAmount, decimals);
       return {
         sufficient: balance.gte(required),
-        balance: balance.toString(),
+        balance: ethers.utils.formatUnits(balance, decimals),
         symbol: method.tokenType.toString(),
       };
     } catch {
@@ -342,7 +350,7 @@ export class PaymentMethodService {
     maxGasPriceGwei: number
   ): Promise<{ acceptable: boolean; currentGasPrice: string }> {
     try {
-      const provider = new ethers.providers.JsonRpcProvider(getEvmRpcUrl(chainId));
+      const provider = getOrCreateResilientProvider(chainId, getEvmRpcUrls(chainId));
       const gasPrice = await provider.getGasPrice();
       const gasPriceGwei = parseFloat(ethers.utils.formatUnits(gasPrice, 'gwei'));
 
@@ -441,10 +449,7 @@ export class PaymentMethodService {
           continue;
         }
 
-        if (
-          method.maxSpendPerInterval &&
-          ethers.BigNumber.from(amount).gt(method.maxSpendPerInterval)
-        ) {
+        if (method.maxSpendPerInterval && exceedsLimit(amount, method.maxSpendPerInterval)) {
           attempt.status = 'failed';
           attempt.failureReason = `Amount ${amount} exceeds max spend per interval ${method.maxSpendPerInterval}`;
           attempt.resolvedAt = new Date();
@@ -485,7 +490,7 @@ export class PaymentMethodService {
     }
 
     try {
-      const provider = new ethers.providers.JsonRpcProvider(getEvmRpcUrl(method.chainId));
+      const provider = getOrCreateResilientProvider(method.chainId, getEvmRpcUrls(method.chainId));
       const code = await provider.getCode(method.tokenAddress);
       const newHash = ethers.utils.keccak256(code);
 
@@ -682,10 +687,7 @@ export class PaymentMethodService {
           continue;
         }
 
-        if (
-          method.maxSpendPerInterval &&
-          ethers.BigNumber.from(amount).gt(method.maxSpendPerInterval)
-        ) {
+        if (method.maxSpendPerInterval && exceedsLimit(amount, method.maxSpendPerInterval)) {
           fail(`Amount ${amount} exceeds max spend per interval ${method.maxSpendPerInterval}`);
           continue;
         }

src/services/rpcProvider.ts

@@ -73,6 +73,8 @@ interface CircuitEntry {
   state: CircuitState;
   consecutiveFailures: number;
   openedAt: number | null;
+  /** Consecutive successes observed while HALF-OPEN, used to close the circuit. */
+  halfOpenSuccesses: number;
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -88,13 +90,16 @@ export interface ResilientProviderConfig {
   failureThreshold?: number;
   /** Ms the circuit stays OPEN before transitioning to HALF-OPEN. Default: 30_000. */
   recoveryTimeoutMs?: number;
+  /** Consecutive successes in HALF-OPEN before closing the circuit. Default: 2. */
+  successThreshold?: number;
 }
 
 const DEFAULT_CONFIG: Required<ResilientProviderConfig> = {
   timeoutMs: 10_000,
   jitterMs: 500,
   failureThreshold: 5,
   recoveryTimeoutMs: 30_000,
+  successThreshold: 2,
 };
 
 /** Returns a sensible timeout for EVM chains with higher RPC variance. */
@@ -132,6 +137,15 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
   private readonly _config: Required<ResilientProviderConfig>;
   private readonly _circuits = new Map<string, CircuitEntry>();
 
+  /**
+   * The base ethers provider talks to a single mutable `connection.url`, so
+   * exactly one `send()` may be swapping that URL at a time. Without this
+   * chain, two concurrent calls (e.g. `getBalancesAcrossChains`, which fans
+   * out in parallel) interleave and one of them is sent to the other's
+   * endpoint — which may be a node on an entirely different chain.
+   */
+  private _swapChain: Promise<unknown> = Promise.resolve();
+
   constructor(urls: string[], chainId: number, config: ResilientProviderConfig = {}) {
     super(urls[0], chainId);
     this._urls = urls;
@@ -141,6 +155,7 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
       jitterMs: config.jitterMs ?? DEFAULT_CONFIG.jitterMs,
       failureThreshold: config.failureThreshold ?? DEFAULT_CONFIG.failureThreshold,
       recoveryTimeoutMs: config.recoveryTimeoutMs ?? DEFAULT_CONFIG.recoveryTimeoutMs,
+      successThreshold: config.successThreshold ?? DEFAULT_CONFIG.successThreshold,
     };
 
     // Initialise circuit entries for every URL
@@ -149,6 +164,7 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
         state: 'closed',
         consecutiveFailures: 0,
         openedAt: null,
+        halfOpenSuccesses: 0,
       });
     }
   }
@@ -166,6 +182,7 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
         const elapsed = Date.now() - (circuit.openedAt ?? 0);
         if (elapsed >= this._config.recoveryTimeoutMs) {
           circuit.state = 'half-open';
+          circuit.halfOpenSuccesses = 0;
         } else {
           errors.push({ url, message: new RpcProviderCircuitOpenError(url).message });
           continue;
@@ -178,7 +195,17 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
         return result;
       } catch (err) {
         const message = err instanceof Error ? err.message : String(err);
-        this._recordFailure(url);
+
+        // Only transport-level problems indicate an unhealthy endpoint. A
+        // legitimate JSON-RPC failure (revert, insufficient funds, invalid
+        // params) means the node answered correctly, so counting it towards
+        // the failure threshold would evict a perfectly healthy URL.
+        if (isTransportFailure(err)) {
+          this._recordFailure(url);
+        } else {
+          this._recordSuccess(url);
+        }
+
         errors.push({ url, message });
 
         logger.warn(
@@ -194,21 +221,40 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
 
   private _getCircuit(url: string): CircuitEntry {
     if (!this._circuits.has(url)) {
-      this._circuits.set(url, { state: 'closed', consecutiveFailures: 0, openedAt: null });
+      this._circuits.set(url, {
+        state: 'closed',
+        consecutiveFailures: 0,
+        openedAt: null,
+        halfOpenSuccesses: 0,
+      });
     }
     return this._circuits.get(url)!;
   }
 
   private _recordSuccess(url: string): void {
     const c = this._getCircuit(url);
     c.consecutiveFailures = 0;
-    c.state = 'closed';
     c.openedAt = null;
+
+    if (c.state === 'half-open') {
+      c.halfOpenSuccesses += 1;
+      // Require consecutive successes before trusting a recovered endpoint
+      // again, so a node that flaps does not ping-pong the circuit.
+      if (c.halfOpenSuccesses >= this._config.successThreshold) {
+        c.state = 'closed';
+        c.halfOpenSuccesses = 0;
+      }
+      return;
+    }
+
+    c.state = 'closed';
+    c.halfOpenSuccesses = 0;
   }
 
   private _recordFailure(url: string): void {
     const c = this._getCircuit(url);
     c.consecutiveFailures += 1;
+    c.halfOpenSuccesses = 0;
 
     if (c.state === 'half-open') {
       // Failed probe — reopen
@@ -237,6 +283,39 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), deadline);
 
+    // Serialise access to the shared `connection.url`. Each caller waits for
+    // the previous swap-and-restore to complete before touching it.
+    const run = this._swapChain.then(
+      () => this._sendToUrl(url, method, params, controller.signal, deadline),
+      () => this._sendToUrl(url, method, params, controller.signal, deadline)
+    );
+
+    this._swapChain = run.then(
+      () => undefined,
+      () => undefined
+    );
+
+    try {
+      return await run;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  private async _sendToUrl(
+    url: string,
+    method: string,
+    params: Array<unknown>,
+    signal: AbortSignal,
+    deadline: number
+  ): Promise<unknown> {
+    // The deadline starts when the caller enqueued, so a call that waited
+    // behind another may already be expired. The abort event has already
+    // fired in that case, so fail fast instead of waiting unbounded.
+    if (signal.aborted) {
+      throw new RpcProviderTimeoutError(url, deadline);
+    }
+
     // We need to temporarily swap the underlying connection URL so the base
     // JsonRpcProvider sends to the right endpoint.
     const previousUrl = this.connection.url;
@@ -247,7 +326,7 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
 
       // Race the call against an abort-signal-aware timeout
       const timeoutPromise = new Promise<never>((_, reject) => {
-        controller.signal.addEventListener(
+        signal.addEventListener(
           'abort',
           () => {
             reject(new RpcProviderTimeoutError(url, deadline));
@@ -258,7 +337,6 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
 
       return await Promise.race([callPromise, timeoutPromise]);
     } finally {
-      clearTimeout(timer);
       // Restore URL in case of future calls to this same provider instance
       (this as unknown as { connection: { url: string } }).connection.url = previousUrl;
     }
@@ -281,11 +359,73 @@ export class ResilientJsonRpcProvider extends ethers.providers.JsonRpcProvider {
       entry.state = 'closed';
       entry.consecutiveFailures = 0;
       entry.openedAt = null;
+      entry.halfOpenSuccesses = 0;
     }
     logger.info(`[ResilientJsonRpcProvider] All circuits reset for chain ${this._chainId}`);
   }
 }
 
+/**
+ * True when an error indicates the endpoint itself is unreachable rather than
+ * the request being rejected by a working node.
+ *
+ * This is what separates "this RPC URL is down" from "this transaction
+ * reverted", and it is the difference between a circuit breaker that protects
+ * the user and one that ejects healthy nodes during an incident.
+ */
+export function isTransportFailure(error: unknown): boolean {
+  if (error instanceof RpcProviderTimeoutError) return true;
+  if (error instanceof RpcProviderCircuitOpenError) return false;
+
+  if (!error || typeof error !== 'object') return false;
+
+  const err = error as {
+    code?: unknown;
+    error?: { code?: unknown };
+    message?: unknown;
+    reason?: unknown;
+  };
+
+  // ethers wraps server responses in `error.code` as a numeric JSON-RPC code.
+  // Anything that reaches here with one of those is an application-level
+  // answer from a live node.
+  const codes = [err.code, err.error?.code];
+  if (codes.some((code) => typeof code === 'number' || /^-?\d+$/.test(String(code)))) {
+    return false;
+  }
+
+  const transportCodes = [
+    'TIMEOUT',
+    'ETIMEDOUT',
+    'ESOCKETTIMEDOUT',
+    'ECONNREFUSED',
+    'ECONNRESET',
+    'EHOSTUNREACH',
+    'ENETUNREACH',
+    'EPIPE',
+    'ERR_NETWORK',
+    'NETWORK_ERROR',
+  ];
+  if (codes.some((code) => typeof code === 'string' && transportCodes.includes(code))) {
+    return true;
+  }
+
+  const text = [err.message, err.reason]
+    .filter((part): part is string => typeof part === 'string')
+    .join(' ')
+    .toLowerCase();
+
+  return (
+    text.includes('failed to fetch') ||
+    text.includes('network error') ||
+    text.includes('fetch failed') ||
+    text.includes('econnrefused') ||
+    text.includes('econnreset') ||
+    text.includes('socket hang up') ||
+    text.includes('getaddrinfo')
+  );
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Singleton registry — circuit state persists across calls in the same session
 // ─────────────────────────────────────────────────────────────────────────────