Treat uninitialized wallets as unverified in isConnected

src/server/blockchain/Web3Wallet.js

@@ -20,16 +20,30 @@ import { toChecksumAddress, sha3 } from 'web3-utils'
 import conf from '../server.config'
 import logger from '../../imports/logger'
 import { isNonceError, isFundsError } from '../utils/eth'
-import { retry as retryAsync, withTimeout } from '../utils/async'
+import { retry as retryAsync, withTimeout, delay } from '../utils/async'
 import { type TransactionReceipt } from './blockchain-types'
 
 import { getManager } from '../utils/tx-manager'
 import { sendSlackAlert } from '../../imports/slack'
 import { KMSWallet } from './KMSWallet'
-// import { HttpProviderFactory, WebsocketProvider } from './transport'
+import { HttpProviderFactory } from './transport'
 
 const ADDRESS_ZERO = '0x0000000000000000000000000000000000000000'
 const FUSE_TX_TIMEOUT = 25000 // should be confirmed after max 5 blocks (25sec)
+const INIT_ATTEMPTS = 3 // initial attempt + 2 retries
+const INIT_RETRY_DELAY = 5000 // base backoff delay between init attempts (5s, 10s)
+
+// strip credentials/api keys (userinfo, path, query) from any url in error messages before alerting
+export const sanitizeErrorMessage = error =>
+  String(error?.message ?? error).replace(/\b(?:https?|wss?):\/\/[^\s'",]+/gi, url => {
+    try {
+      const { protocol, hostname } = new URL(url)
+
+      return `${protocol}//${hostname}`
+    } catch {
+      return '[redacted url]'
+    }
+  })
 
 // Force keccak module to load at import time by calling sha3
 // This ensures keccak is loaded synchronously before Jest can tear down
@@ -104,7 +118,8 @@ export class Web3Wallet {
       maxFeePerGas = undefined,
       maxPriorityFeePerGas = undefined,
       faucetTxCost = 150000,
-      gasPrice = undefined
+      gasPrice = undefined,
+      initRetryDelay = INIT_RETRY_DELAY
     } = options || {}
     const ethOpts = ethereum || conf.fuse
     const { network_id: networkId } = ethOpts
@@ -122,7 +137,10 @@ export class Web3Wallet {
     this.networkId = networkId
     this.maxFeePerGas = maxFeePerGas
     this.maxPriorityFeePerGas = maxPriorityFeePerGas
+    this.feeOptions = { maxFeePerGas, maxPriorityFeePerGas }
     this.gasPrice = gasPrice
+    this.initRetryDelay = initRetryDelay
+    this.initAlertSent = false
     this.log = logger.child({ from: `${name}/${this.networkId}` })
     this.kmsWallet = null // Will be initialized if KMS is used
     this._supportsEIP1559 = null // Cached EIP-1559 support check result
@@ -138,13 +156,63 @@ export class Web3Wallet {
     let { _readyPromise } = this
 
     if (!_readyPromise) {
-      _readyPromise = this.init()
+      _readyPromise = this.initWithRetry()
       assign(this, { _readyPromise })
     }
 
     return _readyPromise
   }
 
+  /**
+   * runs init() up to INIT_ATTEMPTS times with backoff. concurrent callers share the in-flight attempt via _readyPromise.
+   * after the last failure a single slack alert is sent and the wallet is left unavailable (resolves false),
+   * the failed promise is not cached so a later initialize() can start a new bounded cycle
+   * @private
+   */
+  async initWithRetry() {
+    const { log, name, networkId, network, initRetryDelay } = this
+    let lastError
+
+    for (let attempt = 1; attempt <= INIT_ATTEMPTS; attempt++) {
+      try {
+        const ready = await this.init()
+
+        this.initAlertSent = false
+        return ready
+      } catch (e) {
+        lastError = e
+
+        if (attempt < INIT_ATTEMPTS) {
+          const retryIn = initRetryDelay * 2 ** (attempt - 1)
+
+          log.warn('WalletInit: init attempt failed, retrying', { attempt, retryIn, name, networkId, error: e.message })
+          await delay(retryIn)
+        }
+      }
+    }
+
+    this._readyPromise = null
+    log.error('WalletInit: wallet unavailable, all init attempts failed', lastError?.message, lastError, {
+      attempts: INIT_ATTEMPTS,
+      name,
+      networkId
+    })
+
+    if (!this.initAlertSent) {
+      this.initAlertSent = true
+
+      await sendSlackAlert({
+        msg: `CRITICAL: AdminWallet init failed after ${INIT_ATTEMPTS} attempts - ${name}`,
+        wallet: name,
+        networkId,
+        network,
+        error: sanitizeErrorMessage(lastError)
+      })
+    }
+
+    return false
+  }
+
   getWeb3TransportProvider(): HttpProvider | WebSocketProvider {
     const { log } = this
     let provider
@@ -158,8 +226,8 @@ export class Web3Wallet {
 
       case 'HttpProvider':
       default:
-        provider = this.ethereum.httpWeb3Provider.split(',')[0]
-        web3Provider = new Web3.providers.HttpProvider(provider, {
+        provider = this.ethereum.httpWeb3Provider
+        web3Provider = HttpProviderFactory.create(provider, {
           timeout: FUSE_TX_TIMEOUT
         })
         break
@@ -227,6 +295,9 @@ export class Web3Wallet {
   async init() {
     const { log } = this
 
+    // Do not expose an identity contract from a previous or incomplete initialization.
+    this.identityContract = undefined
+
     const adminWalletAddress = get(ContractsAddress, `${this.network}.AdminWallet`)
     log.debug('WalletInit: Initializing wallet:', { conf: this.ethereum, adminWalletAddress, name: this.name })
 
@@ -239,6 +310,9 @@ export class Web3Wallet {
       return false
     }
 
+    // reset per-attempt state so a retried init() doesn't duplicate accounts or reuse a stale EIP-1559 check
+    assign(this, { addresses: [], wallets: {}, _supportsEIP1559: null, ...this.feeOptions })
+
     this.txManager = getManager(this.ethereum.network_id)
     this.web3 = new Web3(this.getWeb3TransportProvider(), null, web3Default)
 
@@ -455,7 +529,7 @@ export class Web3Wallet {
         }
       }
 
-      this.identityContract = new this.web3.eth.Contract(
+      const identityContract = new this.web3.eth.Contract(
         IdentityABI.abi,
         get(ContractsAddress, `${this.network}.Identity`, ADDRESS_ZERO),
         { from: this.address }
@@ -506,6 +580,7 @@ export class Web3Wallet {
 
       let nativebalance = await this.web3.eth.getBalance(this.address)
       this.nonce = parseInt(await this.web3.eth.getTransactionCount(this.address))
+      this.identityContract = identityContract
 
       log.debug('WalletInit: AdminWallet Ready:', {
         activeWallets: this.filledAddresses.length,
@@ -519,11 +594,8 @@ export class Web3Wallet {
     } catch (e) {
       log.error('WalletInit: Error initializing wallet', e.message, e)
 
-      await sendSlackAlert({
-        msg: `CRITICAL: AdminWallet init failed - ${e.message} ${this.name}`
-      })
-
-      return false
+      // failure is retried and alerted on by initWithRetry()
+      throw e
     }
 
     return true
@@ -782,6 +854,16 @@ export class Web3Wallet {
   async isConnected(address: string): Promise<boolean> {
     const { log } = this
 
+    // wallet failed (or not yet finished) initialization - it can't confirm the account, so never count it as verified
+    if (!this.identityContract) {
+      log.warn('isConnected: identity contract unavailable, wallet not initialized', {
+        address,
+        wallet: this.name,
+        networkId: this.networkId
+      })
+      return false
+    }
+
     const tx: string = await this.identityContract.methods
       .getWhitelistedRoot(address)
       .call()

src/server/blockchain/__tests__/multiWallet.test.js

@@ -0,0 +1,66 @@
+import Web3 from 'web3'
+import MultiWallet from '../MultiWallet'
+import { CeloAdminWallet } from '../CeloAdminWallet'
+import { XdcAdminWallet } from '../XdcAdminWallet'
+import { Web3Wallet } from '../Web3Wallet'
+import conf from '../../server.config'
+
+const ADDRESS_ZERO = '0x0000000000000000000000000000000000000000'
+const web3 = new Web3()
+const generateWalletAddress = () => web3.eth.accounts.create().address
+
+const mockIdentityContract = whitelistedRoot => ({
+  methods: {
+    getWhitelistedRoot: () => ({ call: () => Promise.resolve(whitelistedRoot) })
+  }
+})
+
+describe('MultiWallet.isConnected', () => {
+  let account
+  let fuseWallet
+  let celoWallet
+  let xdcWallet
+  let multiWallet
+
+  beforeEach(() => {
+    account = generateWalletAddress()
+
+    fuseWallet = new Web3Wallet('FuseTestWallet', conf, { network: 'development' })
+    celoWallet = CeloAdminWallet({})
+    xdcWallet = XdcAdminWallet({}) // never initialized, so has no identityContract
+
+    fuseWallet.identityContract = mockIdentityContract(ADDRESS_ZERO)
+
+    multiWallet = new MultiWallet.constructor({
+      122: fuseWallet,
+      42220: celoWallet,
+      50: xdcWallet
+    })
+  })
+
+  test('should succeed when XDC is uninitialized and Celo verifies the account', async () => {
+    const warnSpy = jest.spyOn(xdcWallet.log, 'warn')
+
+    celoWallet.identityContract = mockIdentityContract(account)
+
+    expect(xdcWallet.identityContract).toBeUndefined()
+    await expect(multiWallet.isConnected(account)).resolves.toBe(true)
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('identity contract unavailable'),
+      expect.objectContaining({ address: account, wallet: 'XdcAdminWallet' })
+    )
+  })
+
+  test('should not authorize when XDC is uninitialized and no other wallet verifies the account', async () => {
+    const warnSpy = jest.spyOn(xdcWallet.log, 'warn')
+
+    celoWallet.identityContract = mockIdentityContract(ADDRESS_ZERO)
+
+    expect(xdcWallet.identityContract).toBeUndefined()
+    await expect(multiWallet.isConnected(account)).resolves.toBe(false)
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('identity contract unavailable'),
+      expect.objectContaining({ address: account, wallet: 'XdcAdminWallet' })
+    )
+  })
+})

src/server/blockchain/__tests__/partialWalletInit.test.js

@@ -0,0 +1,89 @@
+jest.mock('../../db/mongo-db')
+jest.mock('../../../imports/slack', () => ({ sendSlackAlert: jest.fn() }))
+jest.mock('../../utils/tx-manager', () => ({
+  getManager: () => ({ createListIfNotExists: jest.fn().mockResolvedValue() })
+}))
+jest.mock('web3', () => Object.assign(jest.fn(), { providers: jest.requireActual('web3').providers }))
+
+import Web3 from 'web3'
+import MultiWallet from '../MultiWallet'
+import { Web3Wallet } from '../Web3Wallet'
+import conf from '../../server.config'
+
+const ZERO = '0x0000000000000000000000000000000000000000'
+const account = '0x1111111111111111111111111111111111111111'
+const identity = root => ({ methods: { getWhitelistedRoot: () => ({ call: () => Promise.resolve(root) }) } })
+
+test('late initialization failure cannot interfere with or authorize verification', async () => {
+  let rejectBalance
+  let signalLateBalance
+  const lateBalance = new Promise(resolve => {
+    signalLateBalance = resolve
+  })
+  const failedBalance = new Promise((resolve, reject) => {
+    rejectBalance = reject
+  })
+  const getBalance = jest
+    .fn()
+    .mockResolvedValue('1000000000000000000')
+    .mockImplementationOnce(() => Promise.resolve('1000000000000000000'))
+    .mockImplementationOnce(() => Promise.resolve('1000000000000000000'))
+    .mockImplementationOnce(() => Promise.resolve('1000000000000000000'))
+    .mockImplementationOnce(() => {
+      signalLateBalance()
+      return failedBalance
+    })
+  const contract = jest.fn().mockImplementation(() => ({
+    methods: {
+      adminToppingAmount: () => ({ call: () => Promise.resolve('0') }),
+      getWhitelistedRoot: () => ({ call: () => Promise.reject(new Error('XDC unavailable')) })
+    }
+  }))
+  Web3.mockImplementation(() => ({
+    eth: {
+      getBalance,
+      getTransactionCount: jest.fn().mockResolvedValue(0),
+      Contract: contract,
+      accounts: { privateKeyToAccount: () => ({ address: account }), wallet: { add: jest.fn() } }
+    }
+  }))
+
+  const options = { network: 'production-xdc', ethereum: { network_id: 50 }, initRetryDelay: 0 }
+  const xdc = new Web3Wallet(
+    'XdcAdminWallet',
+    { ...conf, env: 'test', mnemonic: null, privateKey: '0x01', topAdminsOnStartup: false },
+    options,
+    false
+  )
+  const celo = new Web3Wallet('CeloAdminWallet', { ...conf, env: 'test' }, options, false)
+  celo.identityContract = identity(account)
+  xdc.identityContract = identity(account) // stale result from a previous initialization
+  const multi = new MultiWallet.constructor({ 50: xdc, 42220: celo })
+  jest.spyOn(xdc, 'getWeb3TransportProvider').mockReturnValue({})
+  jest.spyOn(xdc, 'supportsEIP1559').mockResolvedValue(true)
+  jest.spyOn(xdc, 'isVerifiedAdmin').mockResolvedValue(true)
+
+  const ready = xdc.ready
+  await lateBalance
+  expect(contract.mock.calls.length).toBeGreaterThanOrEqual(3)
+  expect(xdc.identityContract).toBeUndefined()
+  await expect(multi.isConnected(account)).resolves.toBe(true)
+  rejectBalance(new Error('late RPC failure'))
+  // The subsequent attempts fail earlier, before creating contracts.
+  getBalance.mockImplementation(() => Promise.reject(new Error('RPC unavailable')))
+  await expect(ready).resolves.toBe(false)
+  await expect(multi.isConnected(account)).resolves.toBe(true)
+  celo.identityContract = identity(ZERO)
+  await expect(multi.isConnected(account)).resolves.toBe(false)
+
+  // A later successful initialization publishes XDC's identity for verification.
+  getBalance.mockResolvedValue('1000000000000000000')
+  contract.mockImplementation(() => ({
+    methods: {
+      adminToppingAmount: () => ({ call: () => Promise.resolve('0') }),
+      getWhitelistedRoot: () => ({ call: () => Promise.resolve(account) })
+    }
+  }))
+  await expect(xdc.ready).resolves.toBe(true)
+  await expect(multi.isConnected(account)).resolves.toBe(true)
+})