From 915c932cb2b6f650e603bca9b6f03a563094f7cc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:53:57 +0000 Subject: [PATCH 1/4] Initial plan From ab1cf065944bb40b87904c29f33a0816003a321f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:56:50 +0000 Subject: [PATCH 2/4] Handle uninitialized wallet in isConnected so it can't break shared verification check Co-authored-by: L03TJ3 <6606028+L03TJ3@users.noreply.github.com> --- src/server/blockchain/Web3Wallet.js | 10 +++ .../blockchain/__tests__/multiWallet.test.js | 66 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/server/blockchain/__tests__/multiWallet.test.js diff --git a/src/server/blockchain/Web3Wallet.js b/src/server/blockchain/Web3Wallet.js index 5ddd899ef..1d474f561 100644 --- a/src/server/blockchain/Web3Wallet.js +++ b/src/server/blockchain/Web3Wallet.js @@ -782,6 +782,16 @@ export class Web3Wallet { async isConnected(address: string): Promise { const { log } = this + // wallet failed (or not yet finished) initialization - it can't confirm the account, so never count it as verified + if (!this.identityContract) { + log.warn('isConnected: identity contract unavailable, wallet not initialized', { + address, + wallet: this.name, + networkId: this.networkId + }) + return false + } + const tx: string = await this.identityContract.methods .getWhitelistedRoot(address) .call() diff --git a/src/server/blockchain/__tests__/multiWallet.test.js b/src/server/blockchain/__tests__/multiWallet.test.js new file mode 100644 index 000000000..ec76c4160 --- /dev/null +++ b/src/server/blockchain/__tests__/multiWallet.test.js @@ -0,0 +1,66 @@ +import Web3 from 'web3' +import MultiWallet from '../MultiWallet' +import { CeloAdminWallet } from '../CeloAdminWallet' +import { XdcAdminWallet } from '../XdcAdminWallet' +import { Web3Wallet } from '../Web3Wallet' +import conf from '../../server.config' + +const ADDRESS_ZERO = '0x0000000000000000000000000000000000000000' +const web3 = new Web3() +const generateWalletAddress = () => web3.eth.accounts.create().address + +const mockIdentityContract = whitelistedRoot => ({ + methods: { + getWhitelistedRoot: () => ({ call: () => Promise.resolve(whitelistedRoot) }) + } +}) + +describe('MultiWallet.isConnected', () => { + let account + let fuseWallet + let celoWallet + let xdcWallet + let multiWallet + + beforeEach(() => { + account = generateWalletAddress() + + fuseWallet = new Web3Wallet('FuseTestWallet', conf, { network: 'development' }) + celoWallet = CeloAdminWallet({}) + xdcWallet = XdcAdminWallet({}) // never initialized, so has no identityContract + + fuseWallet.identityContract = mockIdentityContract(ADDRESS_ZERO) + + multiWallet = new MultiWallet.constructor({ + 122: fuseWallet, + 42220: celoWallet, + 50: xdcWallet + }) + }) + + test('should succeed when XDC is uninitialized and Celo verifies the account', async () => { + const warnSpy = jest.spyOn(xdcWallet.log, 'warn') + + celoWallet.identityContract = mockIdentityContract(account) + + expect(xdcWallet.identityContract).toBeUndefined() + await expect(multiWallet.isConnected(account)).resolves.toBe(true) + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('identity contract unavailable'), + expect.objectContaining({ address: account, wallet: 'XdcAdminWallet' }) + ) + }) + + test('should not authorize when XDC is uninitialized and no other wallet verifies the account', async () => { + const warnSpy = jest.spyOn(xdcWallet.log, 'warn') + + celoWallet.identityContract = mockIdentityContract(ADDRESS_ZERO) + + expect(xdcWallet.identityContract).toBeUndefined() + await expect(multiWallet.isConnected(account)).resolves.toBe(false) + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('identity contract unavailable'), + expect.objectContaining({ address: account, wallet: 'XdcAdminWallet' }) + ) + }) +}) From 192b78fbdbc3fe241d205db5ecaeb06cb44b772e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:15:13 +0000 Subject: [PATCH 3/4] Use multi-RPC provider for wallets, retry wallet init 3x with one sanitized Slack alert Co-authored-by: L03TJ3 <6606028+L03TJ3@users.noreply.github.com> --- src/server/blockchain/Web3Wallet.js | 90 ++++++++-- .../blockchain/__tests__/walletInit.test.js | 166 ++++++++++++++++++ src/server/blockchain/transport.js | 9 +- 3 files changed, 252 insertions(+), 13 deletions(-) create mode 100644 src/server/blockchain/__tests__/walletInit.test.js diff --git a/src/server/blockchain/Web3Wallet.js b/src/server/blockchain/Web3Wallet.js index 1d474f561..7ce07ce02 100644 --- a/src/server/blockchain/Web3Wallet.js +++ b/src/server/blockchain/Web3Wallet.js @@ -20,16 +20,30 @@ import { toChecksumAddress, sha3 } from 'web3-utils' import conf from '../server.config' import logger from '../../imports/logger' import { isNonceError, isFundsError } from '../utils/eth' -import { retry as retryAsync, withTimeout } from '../utils/async' +import { retry as retryAsync, withTimeout, delay } from '../utils/async' import { type TransactionReceipt } from './blockchain-types' import { getManager } from '../utils/tx-manager' import { sendSlackAlert } from '../../imports/slack' import { KMSWallet } from './KMSWallet' -// import { HttpProviderFactory, WebsocketProvider } from './transport' +import { HttpProviderFactory } from './transport' const ADDRESS_ZERO = '0x0000000000000000000000000000000000000000' const FUSE_TX_TIMEOUT = 25000 // should be confirmed after max 5 blocks (25sec) +const INIT_ATTEMPTS = 3 // initial attempt + 2 retries +const INIT_RETRY_DELAY = 5000 // base backoff delay between init attempts (5s, 10s) + +// strip credentials/api keys (userinfo, path, query) from any url in error messages before alerting +export const sanitizeErrorMessage = error => + String(error?.message ?? error).replace(/\b(?:https?|wss?):\/\/[^\s'",]+/gi, url => { + try { + const { protocol, hostname } = new URL(url) + + return `${protocol}//${hostname}` + } catch { + return '[redacted url]' + } + }) // Force keccak module to load at import time by calling sha3 // This ensures keccak is loaded synchronously before Jest can tear down @@ -104,7 +118,8 @@ export class Web3Wallet { maxFeePerGas = undefined, maxPriorityFeePerGas = undefined, faucetTxCost = 150000, - gasPrice = undefined + gasPrice = undefined, + initRetryDelay = INIT_RETRY_DELAY } = options || {} const ethOpts = ethereum || conf.fuse const { network_id: networkId } = ethOpts @@ -122,7 +137,10 @@ export class Web3Wallet { this.networkId = networkId this.maxFeePerGas = maxFeePerGas this.maxPriorityFeePerGas = maxPriorityFeePerGas + this.feeOptions = { maxFeePerGas, maxPriorityFeePerGas } this.gasPrice = gasPrice + this.initRetryDelay = initRetryDelay + this.initAlertSent = false this.log = logger.child({ from: `${name}/${this.networkId}` }) this.kmsWallet = null // Will be initialized if KMS is used this._supportsEIP1559 = null // Cached EIP-1559 support check result @@ -138,13 +156,63 @@ export class Web3Wallet { let { _readyPromise } = this if (!_readyPromise) { - _readyPromise = this.init() + _readyPromise = this.initWithRetry() assign(this, { _readyPromise }) } return _readyPromise } + /** + * runs init() up to INIT_ATTEMPTS times with backoff. concurrent callers share the in-flight attempt via _readyPromise. + * after the last failure a single slack alert is sent and the wallet is left unavailable (resolves false), + * the failed promise is not cached so a later initialize() can start a new bounded cycle + * @private + */ + async initWithRetry() { + const { log, name, networkId, network, initRetryDelay } = this + let lastError + + for (let attempt = 1; attempt <= INIT_ATTEMPTS; attempt++) { + try { + const ready = await this.init() + + this.initAlertSent = false + return ready + } catch (e) { + lastError = e + + if (attempt < INIT_ATTEMPTS) { + const retryIn = initRetryDelay * 2 ** (attempt - 1) + + log.warn('WalletInit: init attempt failed, retrying', { attempt, retryIn, name, networkId, error: e.message }) + await delay(retryIn) + } + } + } + + this._readyPromise = null + log.error('WalletInit: wallet unavailable, all init attempts failed', lastError?.message, lastError, { + attempts: INIT_ATTEMPTS, + name, + networkId + }) + + if (!this.initAlertSent) { + this.initAlertSent = true + + await sendSlackAlert({ + msg: `CRITICAL: AdminWallet init failed after ${INIT_ATTEMPTS} attempts - ${name}`, + wallet: name, + networkId, + network, + error: sanitizeErrorMessage(lastError) + }) + } + + return false + } + getWeb3TransportProvider(): HttpProvider | WebSocketProvider { const { log } = this let provider @@ -158,8 +226,8 @@ export class Web3Wallet { case 'HttpProvider': default: - provider = this.ethereum.httpWeb3Provider.split(',')[0] - web3Provider = new Web3.providers.HttpProvider(provider, { + provider = this.ethereum.httpWeb3Provider + web3Provider = HttpProviderFactory.create(provider, { timeout: FUSE_TX_TIMEOUT }) break @@ -239,6 +307,9 @@ export class Web3Wallet { return false } + // reset per-attempt state so a retried init() doesn't duplicate accounts or reuse a stale EIP-1559 check + assign(this, { addresses: [], wallets: {}, _supportsEIP1559: null, ...this.feeOptions }) + this.txManager = getManager(this.ethereum.network_id) this.web3 = new Web3(this.getWeb3TransportProvider(), null, web3Default) @@ -519,11 +590,8 @@ export class Web3Wallet { } catch (e) { log.error('WalletInit: Error initializing wallet', e.message, e) - await sendSlackAlert({ - msg: `CRITICAL: AdminWallet init failed - ${e.message} ${this.name}` - }) - - return false + // failure is retried and alerted on by initWithRetry() + throw e } return true diff --git a/src/server/blockchain/__tests__/walletInit.test.js b/src/server/blockchain/__tests__/walletInit.test.js new file mode 100644 index 000000000..1dd2c9316 --- /dev/null +++ b/src/server/blockchain/__tests__/walletInit.test.js @@ -0,0 +1,166 @@ +// prevent MongoDB connection on import (see src/server/db/__mocks__/mongo-db.js) +jest.mock('../../db/mongo-db') +jest.mock('../../../imports/slack', () => ({ sendSlackAlert: jest.fn() })) + +import http from 'http' +import Web3 from 'web3' +import { sendSlackAlert } from '../../../imports/slack' +import MultiWallet from '../MultiWallet' +import { Web3Wallet } from '../Web3Wallet' +import { MultipleHttpProvider } from '../transport' +import conf from '../../server.config' + +const ADDRESS_ZERO = '0x0000000000000000000000000000000000000000' +const web3 = new Web3() +const generateWalletAddress = () => web3.eth.accounts.create().address + +const mockIdentityContract = whitelistedRoot => ({ + methods: { + getWhitelistedRoot: () => ({ call: () => Promise.resolve(whitelistedRoot) }) + } +}) + +// minimal json-rpc server; `handler` returns the json-rpc response fields for a request +const startRpcServer = handler => + new Promise(resolve => { + const server = http.createServer((req, res) => { + let body = '' + + req.on('data', chunk => (body += chunk)) + req.on('end', () => { + const { id, jsonrpc } = JSON.parse(body) + + server.requests += 1 + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ id, jsonrpc, ...handler() })) + }) + }) + + server.requests = 0 + server.listen(0, '127.0.0.1', () => { + server.url = `http://127.0.0.1:${server.address().port}/` + resolve(server) + }) + }) + +const rateLimited = () => ({ error: { code: -32005, message: 'too many requests, rate limit exceeded' } }) +const chainId = () => ({ result: '0x32' }) + +describe('Web3Wallet RPC endpoint fallback', () => { + const servers = [] + + afterAll(() => Promise.all(servers.map(server => new Promise(resolve => server.close(resolve))))) + + const createWallet = httpWeb3Provider => + new Web3Wallet('XdcAdminWallet', conf, { + ethereum: { network_id: 50, web3Transport: 'HttpProvider', httpWeb3Provider } + }) + + test('should use all configured endpoints and fall back from a rate-limited one', async () => { + const limitedServer = await startRpcServer(rateLimited) + const healthyServer = await startRpcServer(chainId) + + servers.push(limitedServer, healthyServer) + + const wallet = createWallet(`${limitedServer.url},${healthyServer.url}`) + const provider = wallet.getWeb3TransportProvider() + + expect(provider).toBeInstanceOf(MultipleHttpProvider) + expect(provider.endpoints.map(_ => _.provider)).toEqual([limitedServer.url, healthyServer.url]) + + await expect(new Web3(provider).eth.getChainId()).resolves.toBe(50) + expect(limitedServer.requests).toBeGreaterThan(0) + expect(healthyServer.requests).toBeGreaterThan(0) + + // only the rate-limited endpoint is temporarily removed, next calls go straight to the healthy one + expect(provider.endpoints.map(_ => _.provider)).toEqual([healthyServer.url]) + + const limitedRequests = limitedServer.requests + + await expect(new Web3(provider).eth.getChainId()).resolves.toBe(50) + expect(limitedServer.requests).toBe(limitedRequests) + }) +}) + +describe('Web3Wallet initialization retry', () => { + const createWallet = (name, networkId) => + new Web3Wallet(name, conf, { + ethereum: { ...conf.fuse, network_id: networkId }, + initRetryDelay: 20 + }) + + beforeEach(() => { + sendSlackAlert.mockClear() + }) + + test('should recover on a later initialization attempt', async () => { + const wallet = createWallet('XdcAdminWallet', 50) + const initSpy = jest + .spyOn(wallet, 'init') + .mockRejectedValueOnce(new Error('too many requests')) + .mockResolvedValueOnce(true) + + // concurrent callers share the same in-flight initialization + const results = await Promise.all([wallet.ready, wallet.ready]) + + expect(results).toEqual([true, true]) + expect(initSpy).toHaveBeenCalledTimes(2) + expect(sendSlackAlert).not.toHaveBeenCalled() + + // successful result is cached, no further attempts + await expect(wallet.ready).resolves.toBe(true) + expect(initSpy).toHaveBeenCalledTimes(2) + }) + + test('should alert once after 3 failed attempts while a healthy chain remains usable', async () => { + const account = generateWalletAddress() + const fuseWallet = createWallet('AdminWallet', 122) + const celoWallet = createWallet('CeloAdminWallet', 42220) + const xdcWallet = createWallet('XdcAdminWallet', 50) + const multiWallet = new MultiWallet.constructor({ 122: fuseWallet, 42220: celoWallet, 50: xdcWallet }) + + fuseWallet.identityContract = mockIdentityContract(ADDRESS_ZERO) + + jest.spyOn(celoWallet, 'init').mockImplementation(async () => { + celoWallet.identityContract = mockIdentityContract(account) + return true + }) + + const rpcUrl = new URL('https://rpc.xdc.org/apikey123?token=abc') + + rpcUrl.username = 'user' + rpcUrl.password = 'secret' + + const xdcInitSpy = jest + .spyOn(xdcWallet, 'init') + .mockRejectedValue(new Error(`too many requests from ${rpcUrl.href}`)) + + const xdcReady = xdcWallet.ready + + // healthy chain is initialized and usable while xdc is still retrying + await expect(celoWallet.ready).resolves.toBe(true) + expect(xdcInitSpy.mock.calls.length).toBeLessThan(3) + await expect(multiWallet.isConnected(account)).resolves.toBe(true) + + await expect(xdcReady).resolves.toBe(false) + expect(xdcInitSpy).toHaveBeenCalledTimes(3) + expect(xdcWallet.identityContract).toBeUndefined() + + expect(sendSlackAlert).toHaveBeenCalledTimes(1) + + const [[alert]] = sendSlackAlert.mock.calls + + expect(alert).toMatchObject({ wallet: 'XdcAdminWallet', networkId: 50 }) + expect(alert.msg).toContain('XdcAdminWallet') + expect(alert.error).toContain('https://rpc.xdc.org') + expect(JSON.stringify(alert)).not.toMatch(/secret|apikey123|token=abc/) + + // failed result isn't cached: a later call starts a new bounded cycle, without alerting again + await expect(xdcWallet.ready).resolves.toBe(false) + expect(xdcInitSpy).toHaveBeenCalledTimes(6) + expect(sendSlackAlert).toHaveBeenCalledTimes(1) + + // xdc stays unavailable without breaking the shared verification check + await expect(multiWallet.isConnected(account)).resolves.toBe(true) + }) +}) diff --git a/src/server/blockchain/transport.js b/src/server/blockchain/transport.js index 0a11f3d9e..9522448cc 100644 --- a/src/server/blockchain/transport.js +++ b/src/server/blockchain/transport.js @@ -83,8 +83,13 @@ export class MultipleHttpProvider extends HttpProvider { log.error('MultiHttpProvider:', errorMessage, exception, { error, provider, payload }) } else if (isRateLimitError(exception)) { log.warn('MultiHttpProvider rate limit error', exception.message, exception, { error, provider }) - endpoints.splice(endpoints.indexOf(item, 1)) - setTimeout(() => endpoints.push(item), 60000) + const index = endpoints.indexOf(item) + + // temporarily remove only the rate-limited endpoint, but never the last remaining one + if (index >= 0 && endpoints.length > 1) { + endpoints.splice(index, 1) + setTimeout(() => endpoints.push(item), 60000) + } } else if (notTxError) { log.warn('MultiHttpProvider failed to send:', error, exception, { provider }) } From afaf58e1eeadcdb6b7ffc43238d8a5816089e3ac Mon Sep 17 00:00:00 2001 From: LewisB Date: Tue, 27 Oct 2026 23:47:51 +0700 Subject: [PATCH 4/4] fix: not keep identity contract when no admin wallets are initialized for a chain --- src/server/blockchain/Web3Wallet.js | 6 +- .../__tests__/partialWalletInit.test.js | 89 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 src/server/blockchain/__tests__/partialWalletInit.test.js diff --git a/src/server/blockchain/Web3Wallet.js b/src/server/blockchain/Web3Wallet.js index 7ce07ce02..d8b1c7dd6 100644 --- a/src/server/blockchain/Web3Wallet.js +++ b/src/server/blockchain/Web3Wallet.js @@ -295,6 +295,9 @@ export class Web3Wallet { async init() { const { log } = this + // Do not expose an identity contract from a previous or incomplete initialization. + this.identityContract = undefined + const adminWalletAddress = get(ContractsAddress, `${this.network}.AdminWallet`) log.debug('WalletInit: Initializing wallet:', { conf: this.ethereum, adminWalletAddress, name: this.name }) @@ -526,7 +529,7 @@ export class Web3Wallet { } } - this.identityContract = new this.web3.eth.Contract( + const identityContract = new this.web3.eth.Contract( IdentityABI.abi, get(ContractsAddress, `${this.network}.Identity`, ADDRESS_ZERO), { from: this.address } @@ -577,6 +580,7 @@ export class Web3Wallet { let nativebalance = await this.web3.eth.getBalance(this.address) this.nonce = parseInt(await this.web3.eth.getTransactionCount(this.address)) + this.identityContract = identityContract log.debug('WalletInit: AdminWallet Ready:', { activeWallets: this.filledAddresses.length, diff --git a/src/server/blockchain/__tests__/partialWalletInit.test.js b/src/server/blockchain/__tests__/partialWalletInit.test.js new file mode 100644 index 000000000..a54611aed --- /dev/null +++ b/src/server/blockchain/__tests__/partialWalletInit.test.js @@ -0,0 +1,89 @@ +jest.mock('../../db/mongo-db') +jest.mock('../../../imports/slack', () => ({ sendSlackAlert: jest.fn() })) +jest.mock('../../utils/tx-manager', () => ({ + getManager: () => ({ createListIfNotExists: jest.fn().mockResolvedValue() }) +})) +jest.mock('web3', () => Object.assign(jest.fn(), { providers: jest.requireActual('web3').providers })) + +import Web3 from 'web3' +import MultiWallet from '../MultiWallet' +import { Web3Wallet } from '../Web3Wallet' +import conf from '../../server.config' + +const ZERO = '0x0000000000000000000000000000000000000000' +const account = '0x1111111111111111111111111111111111111111' +const identity = root => ({ methods: { getWhitelistedRoot: () => ({ call: () => Promise.resolve(root) }) } }) + +test('late initialization failure cannot interfere with or authorize verification', async () => { + let rejectBalance + let signalLateBalance + const lateBalance = new Promise(resolve => { + signalLateBalance = resolve + }) + const failedBalance = new Promise((resolve, reject) => { + rejectBalance = reject + }) + const getBalance = jest + .fn() + .mockResolvedValue('1000000000000000000') + .mockImplementationOnce(() => Promise.resolve('1000000000000000000')) + .mockImplementationOnce(() => Promise.resolve('1000000000000000000')) + .mockImplementationOnce(() => Promise.resolve('1000000000000000000')) + .mockImplementationOnce(() => { + signalLateBalance() + return failedBalance + }) + const contract = jest.fn().mockImplementation(() => ({ + methods: { + adminToppingAmount: () => ({ call: () => Promise.resolve('0') }), + getWhitelistedRoot: () => ({ call: () => Promise.reject(new Error('XDC unavailable')) }) + } + })) + Web3.mockImplementation(() => ({ + eth: { + getBalance, + getTransactionCount: jest.fn().mockResolvedValue(0), + Contract: contract, + accounts: { privateKeyToAccount: () => ({ address: account }), wallet: { add: jest.fn() } } + } + })) + + const options = { network: 'production-xdc', ethereum: { network_id: 50 }, initRetryDelay: 0 } + const xdc = new Web3Wallet( + 'XdcAdminWallet', + { ...conf, env: 'test', mnemonic: null, privateKey: '0x01', topAdminsOnStartup: false }, + options, + false + ) + const celo = new Web3Wallet('CeloAdminWallet', { ...conf, env: 'test' }, options, false) + celo.identityContract = identity(account) + xdc.identityContract = identity(account) // stale result from a previous initialization + const multi = new MultiWallet.constructor({ 50: xdc, 42220: celo }) + jest.spyOn(xdc, 'getWeb3TransportProvider').mockReturnValue({}) + jest.spyOn(xdc, 'supportsEIP1559').mockResolvedValue(true) + jest.spyOn(xdc, 'isVerifiedAdmin').mockResolvedValue(true) + + const ready = xdc.ready + await lateBalance + expect(contract.mock.calls.length).toBeGreaterThanOrEqual(3) + expect(xdc.identityContract).toBeUndefined() + await expect(multi.isConnected(account)).resolves.toBe(true) + rejectBalance(new Error('late RPC failure')) + // The subsequent attempts fail earlier, before creating contracts. + getBalance.mockImplementation(() => Promise.reject(new Error('RPC unavailable'))) + await expect(ready).resolves.toBe(false) + await expect(multi.isConnected(account)).resolves.toBe(true) + celo.identityContract = identity(ZERO) + await expect(multi.isConnected(account)).resolves.toBe(false) + + // A later successful initialization publishes XDC's identity for verification. + getBalance.mockResolvedValue('1000000000000000000') + contract.mockImplementation(() => ({ + methods: { + adminToppingAmount: () => ({ call: () => Promise.resolve('0') }), + getWhitelistedRoot: () => ({ call: () => Promise.resolve(account) }) + } + })) + await expect(xdc.ready).resolves.toBe(true) + await expect(multi.isConnected(account)).resolves.toBe(true) +})